You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

server.go 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570
  1. package lfs
  2. import (
  3. "encoding/base64"
  4. "encoding/json"
  5. "fmt"
  6. "io"
  7. "net/http"
  8. "path"
  9. "regexp"
  10. "strconv"
  11. "strings"
  12. "time"
  13. "code.gitea.io/gitea/models"
  14. "code.gitea.io/gitea/modules/context"
  15. "code.gitea.io/gitea/modules/log"
  16. "code.gitea.io/gitea/modules/setting"
  17. "github.com/dgrijalva/jwt-go"
  18. "gopkg.in/macaron.v1"
  19. )
  20. const (
  21. contentMediaType = "application/vnd.git-lfs"
  22. metaMediaType = contentMediaType + "+json"
  23. )
  24. // RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and
  25. // some headers are stored.
  26. type RequestVars struct {
  27. Oid string
  28. Size int64
  29. User string
  30. Password string
  31. Repo string
  32. Authorization string
  33. }
  34. // BatchVars contains multiple RequestVars processed in one batch operation.
  35. // https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
  36. type BatchVars struct {
  37. Transfers []string `json:"transfers,omitempty"`
  38. Operation string `json:"operation"`
  39. Objects []*RequestVars `json:"objects"`
  40. }
  41. // BatchResponse contains multiple object metadata Representation structures
  42. // for use with the batch API.
  43. type BatchResponse struct {
  44. Transfer string `json:"transfer,omitempty"`
  45. Objects []*Representation `json:"objects"`
  46. }
  47. // Representation is object metadata as seen by clients of the lfs server.
  48. type Representation struct {
  49. Oid string `json:"oid"`
  50. Size int64 `json:"size"`
  51. Actions map[string]*link `json:"actions"`
  52. Error *ObjectError `json:"error,omitempty"`
  53. }
  54. // ObjectError defines the JSON structure returned to the client in case of an error
  55. type ObjectError struct {
  56. Code int `json:"code"`
  57. Message string `json:"message"`
  58. }
  59. // ObjectLink builds a URL linking to the object.
  60. func (v *RequestVars) ObjectLink() string {
  61. return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/objects", v.Oid)
  62. }
  63. // VerifyLink builds a URL for verifying the object.
  64. func (v *RequestVars) VerifyLink() string {
  65. return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/verify")
  66. }
  67. // link provides a structure used to build a hypermedia representation of an HTTP link.
  68. type link struct {
  69. Href string `json:"href"`
  70. Header map[string]string `json:"header,omitempty"`
  71. ExpiresAt time.Time `json:"expires_at,omitempty"`
  72. }
  73. // ObjectOidHandler is the main request routing entry point into LFS server functions
  74. func ObjectOidHandler(ctx *context.Context) {
  75. if !setting.LFS.StartServer {
  76. writeStatus(ctx, 404)
  77. return
  78. }
  79. if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" {
  80. if MetaMatcher(ctx.Req) {
  81. getMetaHandler(ctx)
  82. return
  83. }
  84. if ContentMatcher(ctx.Req) || len(ctx.Params("filename")) > 0 {
  85. getContentHandler(ctx)
  86. return
  87. }
  88. } else if ctx.Req.Method == "PUT" && ContentMatcher(ctx.Req) {
  89. PutHandler(ctx)
  90. return
  91. }
  92. }
  93. func getAuthenticatedRepoAndMeta(ctx *context.Context, rv *RequestVars, requireWrite bool) (*models.LFSMetaObject, *models.Repository) {
  94. repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
  95. if err != nil {
  96. log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err)
  97. writeStatus(ctx, 404)
  98. return nil, nil
  99. }
  100. if !authenticate(ctx, repository, rv.Authorization, requireWrite) {
  101. requireAuth(ctx)
  102. return nil, nil
  103. }
  104. meta, err := repository.GetLFSMetaObjectByOid(rv.Oid)
  105. if err != nil {
  106. writeStatus(ctx, 404)
  107. return nil, nil
  108. }
  109. return meta, repository
  110. }
  111. // getContentHandler gets the content from the content store
  112. func getContentHandler(ctx *context.Context) {
  113. rv := unpack(ctx)
  114. meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false)
  115. if meta == nil {
  116. return
  117. }
  118. // Support resume download using Range header
  119. var fromByte int64
  120. statusCode := 200
  121. if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" {
  122. regex := regexp.MustCompile(`bytes=(\d+)\-.*`)
  123. match := regex.FindStringSubmatch(rangeHdr)
  124. if match != nil && len(match) > 1 {
  125. statusCode = 206
  126. fromByte, _ = strconv.ParseInt(match[1], 10, 32)
  127. ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, meta.Size-1, meta.Size-fromByte))
  128. }
  129. }
  130. contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
  131. content, err := contentStore.Get(meta, fromByte)
  132. if err != nil {
  133. writeStatus(ctx, 404)
  134. return
  135. }
  136. ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(meta.Size-fromByte, 10))
  137. ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
  138. filename := ctx.Params("filename")
  139. if len(filename) > 0 {
  140. decodedFilename, err := base64.RawURLEncoding.DecodeString(filename)
  141. if err == nil {
  142. ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"")
  143. }
  144. }
  145. ctx.Resp.WriteHeader(statusCode)
  146. io.Copy(ctx.Resp, content)
  147. content.Close()
  148. logRequest(ctx.Req, statusCode)
  149. }
  150. // getMetaHandler retrieves metadata about the object
  151. func getMetaHandler(ctx *context.Context) {
  152. rv := unpack(ctx)
  153. meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false)
  154. if meta == nil {
  155. return
  156. }
  157. ctx.Resp.Header().Set("Content-Type", metaMediaType)
  158. if ctx.Req.Method == "GET" {
  159. enc := json.NewEncoder(ctx.Resp)
  160. enc.Encode(Represent(rv, meta, true, false))
  161. }
  162. logRequest(ctx.Req, 200)
  163. }
  164. // PostHandler instructs the client how to upload data
  165. func PostHandler(ctx *context.Context) {
  166. if !setting.LFS.StartServer {
  167. writeStatus(ctx, 404)
  168. return
  169. }
  170. if !MetaMatcher(ctx.Req) {
  171. writeStatus(ctx, 400)
  172. return
  173. }
  174. rv := unpack(ctx)
  175. repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
  176. if err != nil {
  177. log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err)
  178. writeStatus(ctx, 404)
  179. return
  180. }
  181. if !authenticate(ctx, repository, rv.Authorization, true) {
  182. requireAuth(ctx)
  183. }
  184. meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID})
  185. if err != nil {
  186. writeStatus(ctx, 404)
  187. return
  188. }
  189. ctx.Resp.Header().Set("Content-Type", metaMediaType)
  190. sentStatus := 202
  191. contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
  192. if meta.Existing && contentStore.Exists(meta) {
  193. sentStatus = 200
  194. }
  195. ctx.Resp.WriteHeader(sentStatus)
  196. enc := json.NewEncoder(ctx.Resp)
  197. enc.Encode(Represent(rv, meta, meta.Existing, true))
  198. logRequest(ctx.Req, sentStatus)
  199. }
  200. // BatchHandler provides the batch api
  201. func BatchHandler(ctx *context.Context) {
  202. if !setting.LFS.StartServer {
  203. writeStatus(ctx, 404)
  204. return
  205. }
  206. if !MetaMatcher(ctx.Req) {
  207. writeStatus(ctx, 400)
  208. return
  209. }
  210. bv := unpackbatch(ctx)
  211. var responseObjects []*Representation
  212. // Create a response object
  213. for _, object := range bv.Objects {
  214. repository, err := models.GetRepositoryByOwnerAndName(object.User, object.Repo)
  215. if err != nil {
  216. log.Debug("Could not find repository: %s/%s - %s", object.User, object.Repo, err)
  217. writeStatus(ctx, 404)
  218. return
  219. }
  220. requireWrite := false
  221. if bv.Operation == "upload" {
  222. requireWrite = true
  223. }
  224. if !authenticate(ctx, repository, object.Authorization, requireWrite) {
  225. requireAuth(ctx)
  226. return
  227. }
  228. contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
  229. meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
  230. if err == nil && contentStore.Exists(meta) { // Object is found and exists
  231. responseObjects = append(responseObjects, Represent(object, meta, true, false))
  232. continue
  233. }
  234. // Object is not found
  235. meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
  236. if err == nil {
  237. responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !contentStore.Exists(meta)))
  238. }
  239. }
  240. ctx.Resp.Header().Set("Content-Type", metaMediaType)
  241. respobj := &BatchResponse{Objects: responseObjects}
  242. enc := json.NewEncoder(ctx.Resp)
  243. enc.Encode(respobj)
  244. logRequest(ctx.Req, 200)
  245. }
  246. // PutHandler receives data from the client and puts it into the content store
  247. func PutHandler(ctx *context.Context) {
  248. rv := unpack(ctx)
  249. meta, repository := getAuthenticatedRepoAndMeta(ctx, rv, true)
  250. if meta == nil {
  251. return
  252. }
  253. contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
  254. if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil {
  255. ctx.Resp.WriteHeader(500)
  256. fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
  257. if err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
  258. log.Error(4, "RemoveLFSMetaObjectByOid: %v", err)
  259. }
  260. return
  261. }
  262. logRequest(ctx.Req, 200)
  263. }
  264. // VerifyHandler verify oid and its size from the content store
  265. func VerifyHandler(ctx *context.Context) {
  266. if !setting.LFS.StartServer {
  267. writeStatus(ctx, 404)
  268. return
  269. }
  270. if !ContentMatcher(ctx.Req) {
  271. writeStatus(ctx, 400)
  272. return
  273. }
  274. rv := unpack(ctx)
  275. meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, true)
  276. if meta == nil {
  277. return
  278. }
  279. contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
  280. ok, err := contentStore.Verify(meta)
  281. if err != nil {
  282. ctx.Resp.WriteHeader(500)
  283. fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
  284. return
  285. }
  286. if !ok {
  287. writeStatus(ctx, 422)
  288. return
  289. }
  290. logRequest(ctx.Req, 200)
  291. }
  292. // Represent takes a RequestVars and Meta and turns it into a Representation suitable
  293. // for json encoding
  294. func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation {
  295. rep := &Representation{
  296. Oid: meta.Oid,
  297. Size: meta.Size,
  298. Actions: make(map[string]*link),
  299. }
  300. header := make(map[string]string)
  301. header["Accept"] = contentMediaType
  302. if rv.Authorization == "" {
  303. //https://github.com/github/git-lfs/issues/1088
  304. header["Authorization"] = "Authorization: Basic dummy"
  305. } else {
  306. header["Authorization"] = rv.Authorization
  307. }
  308. if download {
  309. rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header}
  310. }
  311. if upload {
  312. rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header}
  313. }
  314. if upload && !download {
  315. // Force client side verify action while gitea lacks proper server side verification
  316. rep.Actions["verify"] = &link{Href: rv.VerifyLink(), Header: header}
  317. }
  318. return rep
  319. }
  320. // ContentMatcher provides a mux.MatcherFunc that only allows requests that contain
  321. // an Accept header with the contentMediaType
  322. func ContentMatcher(r macaron.Request) bool {
  323. mediaParts := strings.Split(r.Header.Get("Accept"), ";")
  324. mt := mediaParts[0]
  325. return mt == contentMediaType
  326. }
  327. // MetaMatcher provides a mux.MatcherFunc that only allows requests that contain
  328. // an Accept header with the metaMediaType
  329. func MetaMatcher(r macaron.Request) bool {
  330. mediaParts := strings.Split(r.Header.Get("Accept"), ";")
  331. mt := mediaParts[0]
  332. return mt == metaMediaType
  333. }
  334. func unpack(ctx *context.Context) *RequestVars {
  335. r := ctx.Req
  336. rv := &RequestVars{
  337. User: ctx.Params("username"),
  338. Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"),
  339. Oid: ctx.Params("oid"),
  340. Authorization: r.Header.Get("Authorization"),
  341. }
  342. if r.Method == "POST" { // Maybe also check if +json
  343. var p RequestVars
  344. dec := json.NewDecoder(r.Body().ReadCloser())
  345. err := dec.Decode(&p)
  346. if err != nil {
  347. return rv
  348. }
  349. rv.Oid = p.Oid
  350. rv.Size = p.Size
  351. }
  352. return rv
  353. }
  354. // TODO cheap hack, unify with unpack
  355. func unpackbatch(ctx *context.Context) *BatchVars {
  356. r := ctx.Req
  357. var bv BatchVars
  358. dec := json.NewDecoder(r.Body().ReadCloser())
  359. err := dec.Decode(&bv)
  360. if err != nil {
  361. return &bv
  362. }
  363. for i := 0; i < len(bv.Objects); i++ {
  364. bv.Objects[i].User = ctx.Params("username")
  365. bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git")
  366. bv.Objects[i].Authorization = r.Header.Get("Authorization")
  367. }
  368. return &bv
  369. }
  370. func writeStatus(ctx *context.Context, status int) {
  371. message := http.StatusText(status)
  372. mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";")
  373. mt := mediaParts[0]
  374. if strings.HasSuffix(mt, "+json") {
  375. message = `{"message":"` + message + `"}`
  376. }
  377. ctx.Resp.WriteHeader(status)
  378. fmt.Fprint(ctx.Resp, message)
  379. logRequest(ctx.Req, status)
  380. }
  381. func logRequest(r macaron.Request, status int) {
  382. log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status)
  383. }
  384. // authenticate uses the authorization string to determine whether
  385. // or not to proceed. This server assumes an HTTP Basic auth format.
  386. func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool {
  387. accessMode := models.AccessModeRead
  388. if requireWrite {
  389. accessMode = models.AccessModeWrite
  390. }
  391. if !repository.IsPrivate && !requireWrite {
  392. return true
  393. }
  394. if ctx.IsSigned {
  395. accessCheck, _ := models.HasAccess(ctx.User.ID, repository, accessMode)
  396. return accessCheck
  397. }
  398. if authorization == "" {
  399. return false
  400. }
  401. if authenticateToken(repository, authorization, requireWrite) {
  402. return true
  403. }
  404. if !strings.HasPrefix(authorization, "Basic ") {
  405. return false
  406. }
  407. c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic "))
  408. if err != nil {
  409. return false
  410. }
  411. cs := string(c)
  412. i := strings.IndexByte(cs, ':')
  413. if i < 0 {
  414. return false
  415. }
  416. user, password := cs[:i], cs[i+1:]
  417. userModel, err := models.GetUserByName(user)
  418. if err != nil {
  419. return false
  420. }
  421. if !userModel.ValidatePassword(password) {
  422. return false
  423. }
  424. accessCheck, _ := models.HasAccess(userModel.ID, repository, accessMode)
  425. return accessCheck
  426. }
  427. func authenticateToken(repository *models.Repository, authorization string, requireWrite bool) bool {
  428. if !strings.HasPrefix(authorization, "Bearer ") {
  429. return false
  430. }
  431. token, err := jwt.Parse(authorization[7:], func(t *jwt.Token) (interface{}, error) {
  432. if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
  433. return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
  434. }
  435. return setting.LFS.JWTSecretBytes, nil
  436. })
  437. if err != nil {
  438. return false
  439. }
  440. claims, claimsOk := token.Claims.(jwt.MapClaims)
  441. if !token.Valid || !claimsOk {
  442. return false
  443. }
  444. opStr, ok := claims["op"].(string)
  445. if !ok {
  446. return false
  447. }
  448. if requireWrite && opStr != "upload" {
  449. return false
  450. }
  451. repoID, ok := claims["repo"].(float64)
  452. if !ok {
  453. return false
  454. }
  455. if repository.ID != int64(repoID) {
  456. return false
  457. }
  458. return true
  459. }
  460. func requireAuth(ctx *context.Context) {
  461. ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
  462. writeStatus(ctx, 401)
  463. }