Browse Source

Lazy load avatars

Chocobozzz 7 months ago
parent
commit
557b13ae24
No account linked to committer's email address

+ 2
- 0
package.json View File

@@ -130,6 +130,7 @@
130 130
     "jsonld": "~1.1.0",
131 131
     "jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017",
132 132
     "lodash": "^4.17.10",
133
+    "lru-cache": "^5.1.1",
133 134
     "magnet-uri": "^5.1.4",
134 135
     "memoizee": "^0.4.14",
135 136
     "morgan": "^1.5.3",
@@ -179,6 +180,7 @@
179 180
     "@types/fs-extra": "^8.0.0",
180 181
     "@types/libxmljs": "^0.18.0",
181 182
     "@types/lodash": "^4.14.64",
183
+    "@types/lru-cache": "^5.1.0",
182 184
     "@types/magnet-uri": "^5.1.1",
183 185
     "@types/maildev": "^0.0.1",
184 186
     "@types/memoizee": "^0.4.2",

+ 2
- 0
server.ts View File

@@ -97,6 +97,7 @@ import {
97 97
   clientsRouter,
98 98
   feedsRouter,
99 99
   staticRouter,
100
+  lazyStaticRouter,
100 101
   servicesRouter,
101 102
   pluginsRouter,
102 103
   webfingerRouter,
@@ -192,6 +193,7 @@ app.use('/', botsRouter)
192 193
 
193 194
 // Static files
194 195
 app.use('/', staticRouter)
196
+app.use('/', lazyStaticRouter)
195 197
 
196 198
 // Client files, last valid routes!
197 199
 if (cli.client) app.use('/', clientsRouter)

+ 1
- 0
server/controllers/index.ts View File

@@ -4,6 +4,7 @@ export * from './client'
4 4
 export * from './feeds'
5 5
 export * from './services'
6 6
 export * from './static'
7
+export * from './lazy-static'
7 8
 export * from './webfinger'
8 9
 export * from './tracker'
9 10
 export * from './bots'

+ 80
- 0
server/controllers/lazy-static.ts View File

@@ -0,0 +1,80 @@
1
+import * as cors from 'cors'
2
+import * as express from 'express'
3
+import { LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
4
+import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
5
+import { asyncMiddleware } from '../middlewares'
6
+import { AvatarModel } from '../models/avatar/avatar'
7
+import { logger } from '../helpers/logger'
8
+import { avatarPathUnsafeCache, pushAvatarProcessInQueue } from '../lib/avatar'
9
+
10
+const lazyStaticRouter = express.Router()
11
+
12
+lazyStaticRouter.use(cors())
13
+
14
+lazyStaticRouter.use(
15
+  LAZY_STATIC_PATHS.AVATARS + ':filename',
16
+  asyncMiddleware(getAvatar)
17
+)
18
+
19
+lazyStaticRouter.use(
20
+  LAZY_STATIC_PATHS.PREVIEWS + ':uuid.jpg',
21
+  asyncMiddleware(getPreview)
22
+)
23
+
24
+lazyStaticRouter.use(
25
+  LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt',
26
+  asyncMiddleware(getVideoCaption)
27
+)
28
+
29
+// ---------------------------------------------------------------------------
30
+
31
+export {
32
+  lazyStaticRouter,
33
+  getPreview,
34
+  getVideoCaption
35
+}
36
+
37
+// ---------------------------------------------------------------------------
38
+
39
+async function getAvatar (req: express.Request, res: express.Response) {
40
+  const filename = req.params.filename
41
+
42
+  if (avatarPathUnsafeCache.has(filename)) {
43
+    return res.sendFile(avatarPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
44
+  }
45
+
46
+  const avatar = await AvatarModel.loadByName(filename)
47
+  if (avatar.onDisk === false) {
48
+    if (!avatar.fileUrl) return res.sendStatus(404)
49
+
50
+    logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl)
51
+
52
+    await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
53
+
54
+    avatar.onDisk = true
55
+    avatar.save()
56
+      .catch(err => logger.error('Cannot save new avatar disk state.', { err }))
57
+  }
58
+
59
+  const path = avatar.getPath()
60
+
61
+  avatarPathUnsafeCache.set(filename, path)
62
+  return res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER })
63
+}
64
+
65
+async function getPreview (req: express.Request, res: express.Response) {
66
+  const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid)
67
+  if (!result) return res.sendStatus(404)
68
+
69
+  return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER })
70
+}
71
+
72
+async function getVideoCaption (req: express.Request, res: express.Response) {
73
+  const result = await VideosCaptionCache.Instance.getFilePath({
74
+    videoId: req.params.videoId,
75
+    language: req.params.captionLanguage
76
+  })
77
+  if (!result) return res.sendStatus(404)
78
+
79
+  return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER })
80
+}

+ 4
- 20
server/controllers/static.ts View File

@@ -9,7 +9,6 @@ import {
9 9
   STATIC_PATHS,
10 10
   WEBSERVER
11 11
 } from '../initializers/constants'
12
-import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
13 12
 import { cacheRoute } from '../middlewares/cache'
14 13
 import { asyncMiddleware, videosGetValidator } from '../middlewares'
15 14
 import { VideoModel } from '../models/video/video'
@@ -19,6 +18,7 @@ import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/node
19 18
 import { join } from 'path'
20 19
 import { root } from '../helpers/core-utils'
21 20
 import { CONFIG } from '../initializers/config'
21
+import { getPreview, getVideoCaption } from './lazy-static'
22 22
 
23 23
 const staticRouter = express.Router()
24 24
 
@@ -72,19 +72,20 @@ staticRouter.use(
72 72
   express.static(thumbnailsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist
73 73
 )
74 74
 
75
+// DEPRECATED: use lazy-static route instead
75 76
 const avatarsPhysicalPath = CONFIG.STORAGE.AVATARS_DIR
76 77
 staticRouter.use(
77 78
   STATIC_PATHS.AVATARS,
78 79
   express.static(avatarsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist
79 80
 )
80 81
 
81
-// We don't have video previews, fetch them from the origin instance
82
+// DEPRECATED: use lazy-static route instead
82 83
 staticRouter.use(
83 84
   STATIC_PATHS.PREVIEWS + ':uuid.jpg',
84 85
   asyncMiddleware(getPreview)
85 86
 )
86 87
 
87
-// We don't have video captions, fetch them from the origin instance
88
+// DEPRECATED: use lazy-static route instead
88 89
 staticRouter.use(
89 90
   STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt',
90 91
   asyncMiddleware(getVideoCaption)
@@ -177,23 +178,6 @@ export {
177 178
 
178 179
 // ---------------------------------------------------------------------------
179 180
 
180
-async function getPreview (req: express.Request, res: express.Response) {
181
-  const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid)
182
-  if (!result) return res.sendStatus(404)
183
-
184
-  return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE })
185
-}
186
-
187
-async function getVideoCaption (req: express.Request, res: express.Response) {
188
-  const result = await VideosCaptionCache.Instance.getFilePath({
189
-    videoId: req.params.videoId,
190
-    language: req.params.captionLanguage
191
-  })
192
-  if (!result) return res.sendStatus(404)
193
-
194
-  return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE })
195
-}
196
-
197 181
 async function generateNodeinfo (req: express.Request, res: express.Response) {
198 182
   const { totalVideos } = await VideoModel.getStats()
199 183
   const { totalLocalVideoComments } = await VideoCommentModel.getStats()

+ 18
- 4
server/initializers/constants.ts View File

@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
14 14
 
15 15
 // ---------------------------------------------------------------------------
16 16
 
17
-const LAST_MIGRATION_VERSION = 415
17
+const LAST_MIGRATION_VERSION = 420
18 18
 
19 19
 // ---------------------------------------------------------------------------
20 20
 
@@ -498,6 +498,11 @@ const STATIC_DOWNLOAD_PATHS = {
498 498
   TORRENTS: '/download/torrents/',
499 499
   VIDEOS: '/download/videos/'
500 500
 }
501
+const LAZY_STATIC_PATHS = {
502
+  AVATARS: '/lazy-static/avatars/',
503
+  PREVIEWS: '/static/previews/',
504
+  VIDEO_CAPTIONS: '/static/video-captions/'
505
+}
501 506
 
502 507
 // Cache control
503 508
 let STATIC_MAX_AGE = {
@@ -536,9 +541,12 @@ const FILES_CACHE = {
536 541
   }
537 542
 }
538 543
 
539
-const CACHE = {
544
+const LRU_CACHE = {
540 545
   USER_TOKENS: {
541
-    MAX_SIZE: 10000
546
+    MAX_SIZE: 1000
547
+  },
548
+  AVATAR_STATIC: {
549
+    MAX_SIZE: 500
542 550
   }
543 551
 }
544 552
 
@@ -549,6 +557,10 @@ const MEMOIZE_TTL = {
549 557
   OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
550 558
 }
551 559
 
560
+const QUEUE_CONCURRENCY = {
561
+  AVATAR_PROCESS_IMAGE: 3
562
+}
563
+
552 564
 const REDUNDANCY = {
553 565
   VIDEOS: {
554 566
     RANDOMIZED_FACTOR: 5
@@ -649,6 +661,7 @@ export {
649 661
   WEBSERVER,
650 662
   API_VERSION,
651 663
   PEERTUBE_VERSION,
664
+  LAZY_STATIC_PATHS,
652 665
   HLS_REDUNDANCY_DIRECTORY,
653 666
   P2P_MEDIA_LOADER_PEER_VERSION,
654 667
   AVATARS_SIZE,
@@ -695,11 +708,12 @@ export {
695 708
   VIDEO_PRIVACIES,
696 709
   VIDEO_LICENCES,
697 710
   VIDEO_STATES,
711
+  QUEUE_CONCURRENCY,
698 712
   VIDEO_RATE_TYPES,
699 713
   VIDEO_TRANSCODING_FPS,
700 714
   FFMPEG_NICE,
701 715
   VIDEO_ABUSE_STATES,
702
-  CACHE,
716
+  LRU_CACHE,
703 717
   JOB_REQUEST_TIMEOUT,
704 718
   USER_PASSWORD_RESET_LIFETIME,
705 719
   MEMOIZE_TTL,

+ 60
- 0
server/initializers/migrations/0420-avatar-lazy.ts View File

@@ -0,0 +1,60 @@
1
+import * as Sequelize from 'sequelize'
2
+
3
+async function up (utils: {
4
+  transaction: Sequelize.Transaction,
5
+  queryInterface: Sequelize.QueryInterface,
6
+  sequelize: Sequelize.Sequelize,
7
+  db: any
8
+}): Promise<void> {
9
+  {
10
+    // We'll add a unique index on filename, so delete duplicates or PeerTube won't start
11
+    const query = 'DELETE FROM "avatar" s1 ' +
12
+      'USING (SELECT MIN(id) as id, filename FROM "avatar" GROUP BY "filename" HAVING COUNT(*) > 1) s2 ' +
13
+      'WHERE s1."filename" = s2."filename" AND s1.id <> s2.id'
14
+    await utils.sequelize.query(query)
15
+  }
16
+
17
+  {
18
+    const data = {
19
+      type: Sequelize.STRING,
20
+      allowNull: true,
21
+      defaultValue: null
22
+    }
23
+
24
+    await utils.queryInterface.addColumn('avatar', 'fileUrl', data)
25
+  }
26
+
27
+  {
28
+    const data = {
29
+      type: Sequelize.BOOLEAN,
30
+      allowNull: true,
31
+      defaultValue: null
32
+    }
33
+
34
+    await utils.queryInterface.addColumn('avatar', 'onDisk', data)
35
+  }
36
+
37
+  {
38
+    const query = 'UPDATE "avatar" SET "onDisk" = true;'
39
+    await utils.sequelize.query(query)
40
+  }
41
+
42
+  {
43
+    const data = {
44
+      type: Sequelize.BOOLEAN,
45
+      allowNull: false,
46
+      defaultValue: null
47
+    }
48
+
49
+    await utils.queryInterface.changeColumn('avatar', 'onDisk', data)
50
+  }
51
+}
52
+
53
+function down (options) {
54
+  throw new Error('Not implemented.')
55
+}
56
+
57
+export {
58
+  up,
59
+  down
60
+}

+ 40
- 27
server/lib/activitypub/actor.ts View File

@@ -10,9 +10,9 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp
10 10
 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
11 11
 import { logger } from '../../helpers/logger'
12 12
 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
13
-import { doRequest, downloadImage } from '../../helpers/requests'
13
+import { doRequest } from '../../helpers/requests'
14 14
 import { getUrlFromWebfinger } from '../../helpers/webfinger'
15
-import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants'
15
+import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
16 16
 import { AccountModel } from '../../models/account/account'
17 17
 import { ActorModel } from '../../models/activitypub/actor'
18 18
 import { AvatarModel } from '../../models/avatar/avatar'
@@ -21,7 +21,6 @@ import { VideoChannelModel } from '../../models/video/video-channel'
21 21
 import { JobQueue } from '../job-queue'
22 22
 import { getServerActor } from '../../helpers/utils'
23 23
 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
24
-import { CONFIG } from '../../initializers/config'
25 24
 import { sequelizeTypescript } from '../../initializers/database'
26 25
 
27 26
 // Set account keys, this could be long so process after the account creation and do not block the client
@@ -141,25 +140,27 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ
141 140
   actorInstance.followingUrl = attributes.following
142 141
 }
143 142
 
144
-async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
145
-  if (avatarName !== undefined) {
146
-    if (actorInstance.avatarId) {
143
+async function updateActorAvatarInstance (actor: ActorModel, info: { name: string, onDisk: boolean, fileUrl: string }, t: Transaction) {
144
+  if (info.name !== undefined) {
145
+    if (actor.avatarId) {
147 146
       try {
148
-        await actorInstance.Avatar.destroy({ transaction: t })
147
+        await actor.Avatar.destroy({ transaction: t })
149 148
       } catch (err) {
150
-        logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
149
+        logger.error('Cannot remove old avatar of actor %s.', actor.url, { err })
151 150
       }
152 151
     }
153 152
 
154 153
     const avatar = await AvatarModel.create({
155
-      filename: avatarName
154
+      filename: info.name,
155
+      onDisk: info.onDisk,
156
+      fileUrl: info.fileUrl
156 157
     }, { transaction: t })
157 158
 
158
-    actorInstance.set('avatarId', avatar.id)
159
-    actorInstance.Avatar = avatar
159
+    actor.avatarId = avatar.id
160
+    actor.Avatar = avatar
160 161
   }
161 162
 
162
-  return actorInstance
163
+  return actor
163 164
 }
164 165
 
165 166
 async function fetchActorTotalItems (url: string) {
@@ -179,17 +180,17 @@ async function fetchActorTotalItems (url: string) {
179 180
   }
180 181
 }
181 182
 
182
-async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
183
+async function getAvatarInfoIfExists (actorJSON: ActivityPubActor) {
183 184
   if (
184 185
     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
185 186
     isActivityPubUrlValid(actorJSON.icon.url)
186 187
   ) {
187 188
     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
188 189
 
189
-    const avatarName = uuidv4() + extension
190
-    await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
191
-
192
-    return avatarName
190
+    return {
191
+      name: uuidv4() + extension,
192
+      fileUrl: actorJSON.icon.url
193
+    }
193 194
   }
194 195
 
195 196
   return undefined
@@ -245,8 +246,14 @@ async function refreshActorIfNeeded (
245 246
     return sequelizeTypescript.transaction(async t => {
246 247
       updateInstanceWithAnother(actor, result.actor)
247 248
 
248
-      if (result.avatarName !== undefined) {
249
-        await updateActorAvatarInstance(actor, result.avatarName, t)
249
+      if (result.avatar !== undefined) {
250
+        const avatarInfo = {
251
+          name: result.avatar.name,
252
+          fileUrl: result.avatar.fileUrl,
253
+          onDisk: false
254
+        }
255
+
256
+        await updateActorAvatarInstance(actor, avatarInfo, t)
250 257
       }
251 258
 
252 259
       // Force update
@@ -279,7 +286,7 @@ export {
279 286
   buildActorInstance,
280 287
   setAsyncActorKeys,
281 288
   fetchActorTotalItems,
282
-  fetchAvatarIfExists,
289
+  getAvatarInfoIfExists,
283 290
   updateActorInstance,
284 291
   refreshActorIfNeeded,
285 292
   updateActorAvatarInstance,
@@ -314,14 +321,17 @@ function saveActorAndServerAndModelIfNotExist (
314 321
     const [ server ] = await ServerModel.findOrCreate(serverOptions)
315 322
 
316 323
     // Save our new account in database
317
-    actor.set('serverId', server.id)
324
+    actor.serverId = server.id
318 325
 
319 326
     // Avatar?
320
-    if (result.avatarName) {
327
+    if (result.avatar) {
321 328
       const avatar = await AvatarModel.create({
322
-        filename: result.avatarName
329
+        filename: result.avatar.name,
330
+        fileUrl: result.avatar.fileUrl,
331
+        onDisk: false
323 332
       }, { transaction: t })
324
-      actor.set('avatarId', avatar.id)
333
+
334
+      actor.avatarId = avatar.id
325 335
     }
326 336
 
327 337
     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
@@ -355,7 +365,10 @@ type FetchRemoteActorResult = {
355 365
   summary: string
356 366
   support?: string
357 367
   playlists?: string
358
-  avatarName?: string
368
+  avatar?: {
369
+    name: string,
370
+    fileUrl: string
371
+  }
359 372
   attributedTo: ActivityPubAttributedTo[]
360 373
 }
361 374
 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
@@ -399,7 +412,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
399 412
     followingUrl: actorJSON.following
400 413
   })
401 414
 
402
-  const avatarName = await fetchAvatarIfExists(actorJSON)
415
+  const avatarInfo = await getAvatarInfoIfExists(actorJSON)
403 416
 
404 417
   const name = actorJSON.name || actorJSON.preferredUsername
405 418
   return {
@@ -407,7 +420,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
407 420
     result: {
408 421
       actor,
409 422
       name,
410
-      avatarName,
423
+      avatar: avatarInfo,
411 424
       summary: actorJSON.summary,
412 425
       support: actorJSON.support,
413 426
       playlists: actorJSON.playlists,

+ 6
- 4
server/lib/activitypub/process/process-update.ts View File

@@ -6,7 +6,7 @@ import { sequelizeTypescript } from '../../../initializers'
6 6
 import { AccountModel } from '../../../models/account/account'
7 7
 import { ActorModel } from '../../../models/activitypub/actor'
8 8
 import { VideoChannelModel } from '../../../models/video/video-channel'
9
-import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
9
+import { getAvatarInfoIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
10 10
 import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
11 11
 import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
12 12
 import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
@@ -105,7 +105,7 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
105 105
   let accountOrChannelFieldsSave: object
106 106
 
107 107
   // Fetch icon?
108
-  const avatarName = await fetchAvatarIfExists(actorAttributesToUpdate)
108
+  const avatarInfo = await getAvatarInfoIfExists(actorAttributesToUpdate)
109 109
 
110 110
   try {
111 111
     await sequelizeTypescript.transaction(async t => {
@@ -118,8 +118,10 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
118 118
 
119 119
       await updateActorInstance(actor, actorAttributesToUpdate)
120 120
 
121
-      if (avatarName !== undefined) {
122
-        await updateActorAvatarInstance(actor, avatarName, t)
121
+      if (avatarInfo !== undefined) {
122
+        const avatarOptions = Object.assign({}, avatarInfo, { onDisk: false })
123
+
124
+        await updateActorAvatarInstance(actor, avatarOptions, t)
123 125
       }
124 126
 
125 127
       await actor.save({ transaction: t })

+ 35
- 3
server/lib/avatar.ts View File

@@ -1,6 +1,6 @@
1 1
 import 'multer'
2 2
 import { sendUpdateActor } from './activitypub/send'
3
-import { AVATARS_SIZE } from '../initializers/constants'
3
+import { AVATARS_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants'
4 4
 import { updateActorAvatarInstance } from './activitypub'
5 5
 import { processImage } from '../helpers/image-utils'
6 6
 import { AccountModel } from '../models/account/account'
@@ -10,6 +10,9 @@ import { retryTransactionWrapper } from '../helpers/database-utils'
10 10
 import * as uuidv4 from 'uuid/v4'
11 11
 import { CONFIG } from '../initializers/config'
12 12
 import { sequelizeTypescript } from '../initializers/database'
13
+import * as LRUCache from 'lru-cache'
14
+import { queue } from 'async'
15
+import { downloadImage } from '../helpers/requests'
13 16
 
14 17
 async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
15 18
   const extension = extname(avatarPhysicalFile.filename)
@@ -19,7 +22,13 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
19 22
 
20 23
   return retryTransactionWrapper(() => {
21 24
     return sequelizeTypescript.transaction(async t => {
22
-      const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t)
25
+      const avatarInfo = {
26
+        name: avatarName,
27
+        fileUrl: null,
28
+        onDisk: true
29
+      }
30
+
31
+      const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarInfo, t)
23 32
       await updatedActor.save({ transaction: t })
24 33
 
25 34
       await sendUpdateActor(accountOrChannel, t)
@@ -29,6 +38,29 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
29 38
   })
30 39
 }
31 40
 
41
+type DownloadImageQueueTask = { fileUrl: string, filename: string }
42
+
43
+const downloadImageQueue = queue<DownloadImageQueueTask, Error>((task, cb) => {
44
+  downloadImage(task.fileUrl, CONFIG.STORAGE.AVATARS_DIR, task.filename, AVATARS_SIZE)
45
+    .then(() => cb())
46
+    .catch(err => cb(err))
47
+}, QUEUE_CONCURRENCY.AVATAR_PROCESS_IMAGE)
48
+
49
+function pushAvatarProcessInQueue (task: DownloadImageQueueTask) {
50
+  return new Promise((res, rej) => {
51
+    downloadImageQueue.push(task, err => {
52
+      if (err) return rej(err)
53
+
54
+      return res()
55
+    })
56
+  })
57
+}
58
+
59
+// Unsafe so could returns paths that does not exist anymore
60
+const avatarPathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.AVATAR_STATIC.MAX_SIZE })
61
+
32 62
 export {
33
-  updateActorAvatarFile
63
+  avatarPathUnsafeCache,
64
+  updateActorAvatarFile,
65
+  pushAvatarProcessInQueue
34 66
 }

+ 16
- 18
server/lib/oauth-model.ts View File

@@ -4,13 +4,15 @@ import { logger } from '../helpers/logger'
4 4
 import { UserModel } from '../models/account/user'
5 5
 import { OAuthClientModel } from '../models/oauth/oauth-client'
6 6
 import { OAuthTokenModel } from '../models/oauth/oauth-token'
7
-import { CACHE } from '../initializers/constants'
7
+import { LRU_CACHE } from '../initializers/constants'
8 8
 import { Transaction } from 'sequelize'
9 9
 import { CONFIG } from '../initializers/config'
10
+import * as LRUCache from 'lru-cache'
10 11
 
11 12
 type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
12
-let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
13
-let userHavingToken: { [ userId: number ]: string } = {}
13
+
14
+const accessTokenCache = new LRUCache<string, OAuthTokenModel>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
15
+const userHavingToken = new LRUCache<number, string>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
14 16
 
15 17
 // ---------------------------------------------------------------------------
16 18
 
@@ -21,18 +23,20 @@ function deleteUserToken (userId: number, t?: Transaction) {
21 23
 }
22 24
 
23 25
 function clearCacheByUserId (userId: number) {
24
-  const token = userHavingToken[userId]
26
+  const token = userHavingToken.get(userId)
27
+
25 28
   if (token !== undefined) {
26
-    accessTokenCache[ token ] = undefined
27
-    userHavingToken[ userId ] = undefined
29
+    accessTokenCache.del(token)
30
+    userHavingToken.del(userId)
28 31
   }
29 32
 }
30 33
 
31 34
 function clearCacheByToken (token: string) {
32
-  const tokenModel = accessTokenCache[ token ]
35
+  const tokenModel = accessTokenCache.get(token)
36
+
33 37
   if (tokenModel !== undefined) {
34
-    userHavingToken[tokenModel.userId] = undefined
35
-    accessTokenCache[ token ] = undefined
38
+    userHavingToken.del(tokenModel.userId)
39
+    accessTokenCache.del(token)
36 40
   }
37 41
 }
38 42
 
@@ -41,19 +45,13 @@ function getAccessToken (bearerToken: string) {
41 45
 
42 46
   if (!bearerToken) return Bluebird.resolve(undefined)
43 47
 
44
-  if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken])
48
+  if (accessTokenCache.has(bearerToken)) return Bluebird.resolve(accessTokenCache.get(bearerToken))
45 49
 
46 50
   return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
47 51
     .then(tokenModel => {
48 52
       if (tokenModel) {
49
-        // Reinit our cache
50
-        if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) {
51
-          accessTokenCache = {}
52
-          userHavingToken = {}
53
-        }
54
-
55
-        accessTokenCache[ bearerToken ] = tokenModel
56
-        userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
53
+        accessTokenCache.set(bearerToken, tokenModel)
54
+        userHavingToken.set(tokenModel.userId, tokenModel.accessToken)
57 55
       }
58 56
 
59 57
       return tokenModel

+ 2
- 2
server/models/account/user-notification.ts View File

@@ -410,7 +410,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
410 410
         id: this.ActorFollow.ActorFollower.Account.id,
411 411
         displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(),
412 412
         name: this.ActorFollow.ActorFollower.preferredUsername,
413
-        avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined,
413
+        avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getStaticPath() } : undefined,
414 414
         host: this.ActorFollow.ActorFollower.getHost()
415 415
       },
416 416
       following: {
@@ -446,7 +446,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
446 446
 
447 447
   private formatActor (accountOrChannel: AccountModel | VideoChannelModel) {
448 448
     const avatar = accountOrChannel.Actor.Avatar
449
-      ? { path: accountOrChannel.Actor.Avatar.getWebserverPath() }
449
+      ? { path: accountOrChannel.Actor.Avatar.getStaticPath() }
450 450
       : undefined
451 451
 
452 452
     return {

+ 1
- 1
server/models/activitypub/actor.ts View File

@@ -513,7 +513,7 @@ export class ActorModel extends Model<ActorModel> {
513 513
   getAvatarUrl () {
514 514
     if (!this.avatarId) return undefined
515 515
 
516
-    return WEBSERVER.URL + this.Avatar.getWebserverPath()
516
+    return WEBSERVER.URL + this.Avatar.getStaticPath()
517 517
   }
518 518
 
519 519
   isOutdated () {

+ 37
- 6
server/models/avatar/avatar.ts View File

@@ -1,13 +1,21 @@
1 1
 import { join } from 'path'
2
-import { AfterDestroy, AllowNull, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript'
2
+import { AfterDestroy, AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
3 3
 import { Avatar } from '../../../shared/models/avatars/avatar.model'
4
-import { STATIC_PATHS } from '../../initializers/constants'
4
+import { LAZY_STATIC_PATHS } from '../../initializers/constants'
5 5
 import { logger } from '../../helpers/logger'
6 6
 import { remove } from 'fs-extra'
7 7
 import { CONFIG } from '../../initializers/config'
8
+import { throwIfNotValid } from '../utils'
9
+import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
8 10
 
9 11
 @Table({
10
-  tableName: 'avatar'
12
+  tableName: 'avatar',
13
+  indexes: [
14
+    {
15
+      fields: [ 'filename' ],
16
+      unique: true
17
+    }
18
+  ]
11 19
 })
12 20
 export class AvatarModel extends Model<AvatarModel> {
13 21
 
@@ -15,6 +23,15 @@ export class AvatarModel extends Model<AvatarModel> {
15 23
   @Column
16 24
   filename: string
17 25
 
26
+  @AllowNull(true)
27
+  @Is('AvatarFileUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'fileUrl'))
28
+  @Column
29
+  fileUrl: string
30
+
31
+  @AllowNull(false)
32
+  @Column
33
+  onDisk: boolean
34
+
18 35
   @CreatedAt
19 36
   createdAt: Date
20 37
 
@@ -30,16 +47,30 @@ export class AvatarModel extends Model<AvatarModel> {
30 47
       .catch(err => logger.error('Cannot remove avatar file %s.', instance.filename, err))
31 48
   }
32 49
 
50
+  static loadByName (filename: string) {
51
+    const query = {
52
+      where: {
53
+        filename
54
+      }
55
+    }
56
+
57
+    return AvatarModel.findOne(query)
58
+  }
59
+
33 60
   toFormattedJSON (): Avatar {
34 61
     return {
35
-      path: this.getWebserverPath(),
62
+      path: this.getStaticPath(),
36 63
       createdAt: this.createdAt,
37 64
       updatedAt: this.updatedAt
38 65
     }
39 66
   }
40 67
 
41
-  getWebserverPath () {
42
-    return join(STATIC_PATHS.AVATARS, this.filename)
68
+  getStaticPath () {
69
+    return join(LAZY_STATIC_PATHS.AVATARS, this.filename)
70
+  }
71
+
72
+  getPath () {
73
+    return join(CONFIG.STORAGE.AVATARS_DIR, this.filename)
43 74
   }
44 75
 
45 76
   removeAvatar () {

+ 2
- 2
server/models/video/thumbnail.ts View File

@@ -1,6 +1,6 @@
1 1
 import { join } from 'path'
2 2
 import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
3
-import { STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
3
+import { LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
4 4
 import { logger } from '../../helpers/logger'
5 5
 import { remove } from 'fs-extra'
6 6
 import { CONFIG } from '../../initializers/config'
@@ -87,7 +87,7 @@ export class ThumbnailModel extends Model<ThumbnailModel> {
87 87
     [ThumbnailType.PREVIEW]: {
88 88
       label: 'preview',
89 89
       directory: CONFIG.STORAGE.PREVIEWS_DIR,
90
-      staticPath: STATIC_PATHS.PREVIEWS
90
+      staticPath: LAZY_STATIC_PATHS.PREVIEWS
91 91
     }
92 92
   }
93 93
 

+ 2
- 2
server/models/video/video-caption.ts View File

@@ -16,7 +16,7 @@ import { buildWhereIdOrUUID, throwIfNotValid } from '../utils'
16 16
 import { VideoModel } from './video'
17 17
 import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions'
18 18
 import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
19
-import { STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants'
19
+import { LAZY_STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants'
20 20
 import { join } from 'path'
21 21
 import { logger } from '../../helpers/logger'
22 22
 import { remove } from 'fs-extra'
@@ -163,7 +163,7 @@ export class VideoCaptionModel extends Model<VideoCaptionModel> {
163 163
   }
164 164
 
165 165
   getCaptionStaticPath () {
166
-    return join(STATIC_PATHS.VIDEO_CAPTIONS, this.getCaptionName())
166
+    return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.getCaptionName())
167 167
   }
168 168
 
169 169
   getCaptionName () {

+ 2
- 1
server/models/video/video.ts View File

@@ -63,6 +63,7 @@ import {
63 63
   CONSTRAINTS_FIELDS,
64 64
   HLS_REDUNDANCY_DIRECTORY,
65 65
   HLS_STREAMING_PLAYLIST_DIRECTORY,
66
+  LAZY_STATIC_PATHS,
66 67
   REMOTE_SCHEME,
67 68
   STATIC_DOWNLOAD_PATHS,
68 69
   STATIC_PATHS,
@@ -1856,7 +1857,7 @@ export class VideoModel extends Model<VideoModel> {
1856 1857
     if (!preview) return null
1857 1858
 
1858 1859
     // We use a local cache, so specify our cache endpoint instead of potential remote URL
1859
-    return join(STATIC_PATHS.PREVIEWS, preview.filename)
1860
+    return join(LAZY_STATIC_PATHS.PREVIEWS, preview.filename)
1860 1861
   }
1861 1862
 
1862 1863
   toFormattedJSON (options?: VideoFormattingJSONOptions): Video {

+ 13
- 1
yarn.lock View File

@@ -197,6 +197,11 @@
197 197
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.136.tgz#413e85089046b865d960c9ff1d400e04c31ab60f"
198 198
   integrity sha512-0GJhzBdvsW2RUccNHOBkabI8HZVdOXmXbXhuKlDEd5Vv12P7oAVGfomGp3Ne21o5D/qu1WmthlNKFaoZJJeErA==
199 199
 
200
+"@types/lru-cache@^5.1.0":
201
+  version "5.1.0"
202
+  resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03"
203
+  integrity sha512-RaE0B+14ToE4l6UqdarKPnXwVDuigfFv+5j9Dze/Nqr23yyuqdNvzcZi3xB+3Agvi5R4EOgAksfv3lXX4vBt9w==
204
+
200 205
 "@types/magnet-uri@*", "@types/magnet-uri@^5.1.1":
201 206
   version "5.1.2"
202 207
   resolved "https://registry.yarnpkg.com/@types/magnet-uri/-/magnet-uri-5.1.2.tgz#7860417399d52ddc0be1021d570b4ac93ffc133e"
@@ -4394,6 +4399,13 @@ lru-cache@4.1.x, lru-cache@^4.0.1:
4394 4399
     pseudomap "^1.0.2"
4395 4400
     yallist "^2.1.2"
4396 4401
 
4402
+lru-cache@^5.1.1:
4403
+  version "5.1.1"
4404
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920"
4405
+  integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==
4406
+  dependencies:
4407
+    yallist "^3.0.2"
4408
+
4397 4409
 lru-queue@0.1:
4398 4410
   version "0.1.0"
4399 4411
   resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3"
@@ -8082,7 +8094,7 @@ yallist@^2.1.2:
8082 8094
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
8083 8095
   integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=
8084 8096
 
8085
-yallist@^3.0.0, yallist@^3.0.3:
8097
+yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3:
8086 8098
   version "3.0.3"
8087 8099
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.3.tgz#b4b049e314be545e3ce802236d6cd22cd91c3de9"
8088 8100
   integrity sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==

Loading…
Cancel
Save