Redesign Scoped Access Tokens (#24767)

## Changes
- Adds the following high level access scopes, each with `read` and
`write` levels:
    - `activitypub`
    - `admin` (hidden if user is not a site admin)
    - `misc`
    - `notification`
    - `organization`
    - `package`
    - `issue`
    - `repository`
    - `user`
- Adds new middleware function `tokenRequiresScopes()` in addition to
`reqToken()`
  -  `tokenRequiresScopes()` is used for each high-level api section
- _if_ a scoped token is present, checks that the required scope is
included based on the section and HTTP method
  - `reqToken()` is used for individual routes
- checks that required authentication is present (but does not check
scope levels as this will already have been handled by
`tokenRequiresScopes()`
- Adds migration to convert old scoped access tokens to the new set of
scopes
- Updates the user interface for scope selection

### User interface example
<img width="903" alt="Screen Shot 2023-05-31 at 1 56 55 PM"
src="https://github.com/go-gitea/gitea/assets/23248839/654766ec-2143-4f59-9037-3b51600e32f3">
<img width="917" alt="Screen Shot 2023-05-31 at 1 56 43 PM"
src="https://github.com/go-gitea/gitea/assets/23248839/1ad64081-012c-4a73-b393-66b30352654c">

## tokenRequiresScopes  Design Decision
- `tokenRequiresScopes()` was added to more reliably cover api routes.
For an incoming request, this function uses the given scope category
(say `AccessTokenScopeCategoryOrganization`) and the HTTP method (say
`DELETE`) and verifies that any scoped tokens in use include
`delete:organization`.
- `reqToken()` is used to enforce auth for individual routes that
require it. If a scoped token is not present for a request,
`tokenRequiresScopes()` will not return an error

## TODO
- [x] Alphabetize scope categories
- [x] Change 'public repos only' to a radio button (private vs public).
Also expand this to organizations
- [X] Disable token creation if no scopes selected. Alternatively, show
warning
- [x] `reqToken()` is missing from many `POST/DELETE` routes in the api.
`tokenRequiresScopes()` only checks that a given token has the correct
scope, `reqToken()` must be used to check that a token (or some other
auth) is present.
   -  _This should be addressed in this PR_
- [x] The migration should be reviewed very carefully in order to
minimize access changes to existing user tokens.
   - _This should be addressed in this PR_
- [x] Link to api to swagger documentation, clarify what
read/write/delete levels correspond to
- [x] Review cases where more than one scope is needed as this directly
deviates from the api definition.
   - _This should be addressed in this PR_
   - For example: 
   ```go
	m.Group("/users/{username}/orgs", func() {
		m.Get("", reqToken(), org.ListUserOrgs)
		m.Get("/{org}/permissions", reqToken(), org.GetUserOrgsPermissions)
}, tokenRequiresScopes(auth_model.AccessTokenScopeCategoryUser,
auth_model.AccessTokenScopeCategoryOrganization),
context_service.UserAssignmentAPI())
   ```

## Future improvements
- [ ] Add required scopes to swagger documentation
- [ ] Redesign `reqToken()` to be opt-out rather than opt-in
- [ ] Subdivide scopes like `repository`
- [ ] Once a token is created, if it has no scopes, we should display
text instead of an empty bullet point
- [ ] If the 'public repos only' option is selected, should read
categories be selected by default

Closes #24501
Closes #24799

Co-authored-by: Jonathan Tran <jon@allspice.io>
Co-authored-by: Kyle D <kdumontnu@gmail.com>
Co-authored-by: silverwind <me@silverwind.io>

1 files changed, 522 insertions, 26 deletions

diff --git a/tests/integration/api_token_test.go b/tests/integration/api_token_test.go
index 498bd41b0f..0eecccb653 100644
--- a/tests/integration/api_token_test.go
+++ b/tests/integration/api_token_test.go
@@ -4,14 +4,18 @@
 package integration
 
 import (
+	"fmt"
 	"net/http"
 	"testing"
 
 	auth_model "code.gitea.io/gitea/models/auth"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/tests"
+
+	"github.com/stretchr/testify/assert"
 )
 
 // TestAPICreateAndDeleteToken tests that token that was just created can be deleted
@@ -19,9 +23,518 @@ func TestAPICreateAndDeleteToken(t *testing.T) {
 	defer tests.PrepareTestEnv(t)()
 	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
 
-	req := NewRequestWithJSON(t, "POST", "/api/v1/users/user1/tokens", map[string]string{
-		"name": "test-key-1",
+	newAccessToken := createAPIAccessTokenWithoutCleanUp(t, "test-key-1", user, nil)
+	deleteAPIAccessToken(t, newAccessToken, user)
+
+	newAccessToken = createAPIAccessTokenWithoutCleanUp(t, "test-key-2", user, nil)
+	deleteAPIAccessToken(t, newAccessToken, user)
+}
+
+// TestAPIDeleteMissingToken ensures that error is thrown when token not found
+func TestAPIDeleteMissingToken(t *testing.T) {
+	defer tests.PrepareTestEnv(t)()
+	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+
+	req := NewRequestf(t, "DELETE", "/api/v1/users/user1/tokens/%d", unittest.NonexistentID)
+	req = AddBasicAuthHeader(req, user.Name)
+	MakeRequest(t, req, http.StatusNotFound)
+}
+
+type permission struct {
+	category auth_model.AccessTokenScopeCategory
+	level    auth_model.AccessTokenScopeLevel
+}
+
+type requiredScopeTestCase struct {
+	url                 string
+	method              string
+	requiredPermissions []permission
+}
+
+func (c *requiredScopeTestCase) Name() string {
+	return fmt.Sprintf("%v %v", c.method, c.url)
+}
+
+// TestAPIDeniesPermissionBasedOnTokenScope tests that API routes forbid access
+// when the correct token scope is not included.
+func TestAPIDeniesPermissionBasedOnTokenScope(t *testing.T) {
+	defer tests.PrepareTestEnv(t)()
+
+	// We'll assert that each endpoint, when fetched with a token with all
+	// scopes *except* the ones specified, a forbidden status code is returned.
+	//
+	// This is to protect against endpoints having their access check copied
+	// from other endpoints and not updated.
+	//
+	// Test cases are in alphabetical order by URL.
+	//
+	// Note: query parameters are not currently supported since the token is
+	// appended with `?=token=<token>`.
+	testCases := []requiredScopeTestCase{
+		{
+			"/api/v1/admin/emails",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryAdmin,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/admin/users",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryAdmin,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/admin/users",
+			"POST",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryAdmin,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/admin/users/user2",
+			"PATCH",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryAdmin,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/admin/users/user2/orgs",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryAdmin,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/admin/users/user2/orgs",
+			"POST",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryAdmin,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/admin/orgs",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryAdmin,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/markdown",
+			"POST",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryMisc,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/markdown/raw",
+			"POST",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryMisc,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/notifications",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryNotification,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/notifications",
+			"PUT",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryNotification,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/org/org1/repos",
+			"POST",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryOrganization,
+					auth_model.Write,
+				},
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/packages/user1/type/name/1",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryPackage,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/packages/user1/type/name/1",
+			"DELETE",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryPackage,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1",
+			"PATCH",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1",
+			"DELETE",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1/branches",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1/archive/foo",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1/issues",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryIssue,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1/media/foo",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1/raw/foo",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1/teams",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1/teams/team1",
+			"PUT",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/repos/user1/repo1/transfer",
+			"POST",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Write,
+				},
+			},
+		},
+		// Private repo
+		{
+			"/api/v1/repos/user2/repo2",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Read,
+				},
+			},
+		},
+		// Private repo
+		{
+			"/api/v1/repos/user2/repo2",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryRepository,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/settings/api",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryMisc,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/user",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/user/emails",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/user/emails",
+			"POST",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/user/emails",
+			"DELETE",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/user/applications/oauth2",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Read,
+				},
+			},
+		},
+		{
+			"/api/v1/user/applications/oauth2",
+			"POST",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Write,
+				},
+			},
+		},
+		{
+			"/api/v1/users/search",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Read,
+				},
+			},
+		},
+		// Private user
+		{
+			"/api/v1/users/user31",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Read,
+				},
+			},
+		},
+		// Private user
+		{
+			"/api/v1/users/user31/gpg_keys",
+			"GET",
+			[]permission{
+				{
+					auth_model.AccessTokenScopeCategoryUser,
+					auth_model.Read,
+				},
+			},
+		},
+	}
+
+	// User needs to be admin so that we can verify that tokens without admin
+	// scopes correctly deny access.
+	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+	assert.True(t, user.IsAdmin, "User needs to be admin")
+
+	for _, testCase := range testCases {
+		runTestCase(t, &testCase, user)
+	}
+}
+
+// runTestCase Helper function to run a single test case.
+func runTestCase(t *testing.T, testCase *requiredScopeTestCase, user *user_model.User) {
+	t.Run(testCase.Name(), func(t *testing.T) {
+		defer tests.PrintCurrentTest(t)()
+
+		// Create a token with all scopes NOT required by the endpoint.
+		var unauthorizedScopes []auth_model.AccessTokenScope
+		for _, category := range auth_model.AllAccessTokenScopeCategories {
+			// For permissions, Write > Read > NoAccess.  So we need to
+			// find the minimum required, and only grant permission up to but
+			// not including the minimum required.
+			minRequiredLevel := auth_model.Write
+			categoryIsRequired := false
+			for _, requiredPermission := range testCase.requiredPermissions {
+				if requiredPermission.category != category {
+					continue
+				}
+				categoryIsRequired = true
+				if requiredPermission.level < minRequiredLevel {
+					minRequiredLevel = requiredPermission.level
+				}
+			}
+			unauthorizedLevel := auth_model.Write
+			if categoryIsRequired {
+				if minRequiredLevel == auth_model.Read {
+					unauthorizedLevel = auth_model.NoAccess
+				} else if minRequiredLevel == auth_model.Write {
+					unauthorizedLevel = auth_model.Read
+				} else {
+					assert.Failf(t, "Invalid test case", "Unknown access token scope level: %v", minRequiredLevel)
+					return
+				}
+			}
+
+			if unauthorizedLevel == auth_model.NoAccess {
+				continue
+			}
+			cateogoryUnauthorizedScopes := auth_model.GetRequiredScopes(
+				unauthorizedLevel,
+				category)
+			unauthorizedScopes = append(unauthorizedScopes, cateogoryUnauthorizedScopes...)
+		}
+
+		accessToken := createAPIAccessTokenWithoutCleanUp(t, "test-token", user, &unauthorizedScopes)
+		defer deleteAPIAccessToken(t, accessToken, user)
+
+		// Add API access token to the URL.
+		url := fmt.Sprintf("%s?token=%s", testCase.url, accessToken.Token)
+
+		// Request the endpoint.  Verify that permission is denied.
+		req := NewRequestf(t, testCase.method, url)
+		MakeRequest(t, req, http.StatusForbidden)
 	})
+}
+
+// createAPIAccessTokenWithoutCleanUp Create an API access token and assert that
+// creation succeeded.  The caller is responsible for deleting the token.
+func createAPIAccessTokenWithoutCleanUp(t *testing.T, tokenName string, user *user_model.User, scopes *[]auth_model.AccessTokenScope) api.AccessToken {
+	payload := map[string]interface{}{
+		"name": tokenName,
+	}
+	if scopes != nil {
+		for _, scope := range *scopes {
+			scopes, scopesExists := payload["scopes"].([]string)
+			if !scopesExists {
+				scopes = make([]string, 0)
+			}
+			scopes = append(scopes, string(scope))
+			payload["scopes"] = scopes
+		}
+	}
+	log.Debug("Requesting creation of token with scopes: %v", scopes)
+	req := NewRequestWithJSON(t, "POST", "/api/v1/users/user1/tokens", payload)
+
 	req = AddBasicAuthHeader(req, user.Name)
 	resp := MakeRequest(t, req, http.StatusCreated)
 
@@ -34,32 +547,15 @@ func TestAPICreateAndDeleteToken(t *testing.T) {
 		UID:   user.ID,
 	})
 
-	req = NewRequestf(t, "DELETE", "/api/v1/users/user1/tokens/%d", newAccessToken.ID)
-	req = AddBasicAuthHeader(req, user.Name)
-	MakeRequest(t, req, http.StatusNoContent)
-
-	unittest.AssertNotExistsBean(t, &auth_model.AccessToken{ID: newAccessToken.ID})
-
-	req = NewRequestWithJSON(t, "POST", "/api/v1/users/user1/tokens", map[string]string{
-		"name": "test-key-2",
-	})
-	req = AddBasicAuthHeader(req, user.Name)
-	resp = MakeRequest(t, req, http.StatusCreated)
-	DecodeJSON(t, resp, &newAccessToken)
+	return newAccessToken
+}
 
-	req = NewRequestf(t, "DELETE", "/api/v1/users/user1/tokens/%s", newAccessToken.Name)
+// createAPIAccessTokenWithoutCleanUp Delete an API access token and assert that
+// deletion succeeded.
+func deleteAPIAccessToken(t *testing.T, accessToken api.AccessToken, user *user_model.User) {
+	req := NewRequestf(t, "DELETE", "/api/v1/users/user1/tokens/%d", accessToken.ID)
 	req = AddBasicAuthHeader(req, user.Name)
 	MakeRequest(t, req, http.StatusNoContent)
 
-	unittest.AssertNotExistsBean(t, &auth_model.AccessToken{ID: newAccessToken.ID})
-}
-
-// TestAPIDeleteMissingToken ensures that error is thrown when token not found
-func TestAPIDeleteMissingToken(t *testing.T) {
-	defer tests.PrepareTestEnv(t)()
-	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
-
-	req := NewRequestf(t, "DELETE", "/api/v1/users/user1/tokens/%d", unittest.NonexistentID)
-	req = AddBasicAuthHeader(req, user.Name)
-	MakeRequest(t, req, http.StatusNotFound)
+	unittest.AssertNotExistsBean(t, &auth_model.AccessToken{ID: accessToken.ID})
 }