Browse Source

Add auto follow instances index support

Chocobozzz 1 month ago
parent
commit
6f1b4fa417

+ 2
- 0
server.ts View File

@@ -115,6 +115,7 @@ import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-sch
115 115
 import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
116 116
 import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
117 117
 import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
118
+import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
118 119
 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
119 120
 import { PeerTubeSocket } from './server/lib/peertube-socket'
120 121
 import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
@@ -260,6 +261,7 @@ async function startApplication () {
260 261
   RemoveOldHistoryScheduler.Instance.enable()
261 262
   RemoveOldViewsScheduler.Instance.enable()
262 263
   PluginsCheckScheduler.Instance.enable()
264
+  AutoFollowIndexInstances.Instance.enable()
263 265
 
264 266
   // Redis initialization
265 267
   Redis.Instance.init()

+ 7
- 0
server/initializers/constants.ts View File

@@ -168,10 +168,15 @@ const SCHEDULER_INTERVALS_MS = {
168 168
   updateVideos: 60000, // 1 minute
169 169
   youtubeDLUpdate: 60000 * 60 * 24, // 1 day
170 170
   checkPlugins: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL,
171
+  autoFollowIndexInstances: 60000 * 60 * 24, // 1 day
171 172
   removeOldViews: 60000 * 60 * 24, // 1 day
172 173
   removeOldHistory: 60000 * 60 * 24 // 1 day
173 174
 }
174 175
 
176
+const INSTANCES_INDEX = {
177
+  HOSTS_PATH: '/api/v1/instances/hosts'
178
+}
179
+
175 180
 // ---------------------------------------------------------------------------
176 181
 
177 182
 const CONSTRAINTS_FIELDS = {
@@ -633,6 +638,7 @@ if (isTestInstance() === true) {
633 638
   SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
634 639
   SCHEDULER_INTERVALS_MS.removeOldViews = 5000
635 640
   SCHEDULER_INTERVALS_MS.updateVideos = 5000
641
+  SCHEDULER_INTERVALS_MS.autoFollowIndexInstances = 5000
636 642
   REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
637 643
 
638 644
   REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
@@ -683,6 +689,7 @@ export {
683 689
   PREVIEWS_SIZE,
684 690
   REMOTE_SCHEME,
685 691
   FOLLOW_STATES,
692
+  INSTANCES_INDEX,
686 693
   DEFAULT_USER_THEME_NAME,
687 694
   SERVER_ACTOR_NAME,
688 695
   PLUGIN_GLOBAL_CSS_FILE_NAME,

+ 72
- 0
server/lib/schedulers/auto-follow-index-instances.ts View File

@@ -0,0 +1,72 @@
1
+import { logger } from '../../helpers/logger'
2
+import { AbstractScheduler } from './abstract-scheduler'
3
+import { INSTANCES_INDEX, SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants'
4
+import { CONFIG } from '../../initializers/config'
5
+import { chunk } from 'lodash'
6
+import { doRequest } from '@server/helpers/requests'
7
+import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
8
+import { JobQueue } from '@server/lib/job-queue'
9
+import { getServerActor } from '@server/helpers/utils'
10
+
11
+export class AutoFollowIndexInstances extends AbstractScheduler {
12
+
13
+  private static instance: AbstractScheduler
14
+
15
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.autoFollowIndexInstances
16
+
17
+  private lastCheck: Date
18
+
19
+  private constructor () {
20
+    super()
21
+  }
22
+
23
+  protected async internalExecute () {
24
+    return this.autoFollow()
25
+  }
26
+
27
+  private async autoFollow () {
28
+    if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return
29
+
30
+    const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL
31
+
32
+    logger.info('Auto follow instances of index %s.', indexUrl)
33
+
34
+    try {
35
+      const serverActor = await getServerActor()
36
+
37
+      const uri = indexUrl + INSTANCES_INDEX.HOSTS_PATH
38
+
39
+      const qs = this.lastCheck ? { since: this.lastCheck.toISOString() } : {}
40
+      this.lastCheck = new Date()
41
+
42
+      const { body } = await doRequest({ uri, qs, json: true })
43
+
44
+      const hosts: string[] = body.data.map(o => o.host)
45
+      const chunks = chunk(hosts, 20)
46
+
47
+      for (const chunk of chunks) {
48
+        const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk)
49
+
50
+        for (const unfollowedHost of unfollowedHosts) {
51
+          const payload = {
52
+            host: unfollowedHost,
53
+            name: SERVER_ACTOR_NAME,
54
+            followerActorId: serverActor.id,
55
+            isAutoFollow: true
56
+          }
57
+
58
+          await JobQueue.Instance.createJob({ type: 'activitypub-follow', payload })
59
+                  .catch(err => logger.error('Cannot create follow job for %s.', unfollowedHost, err))
60
+        }
61
+      }
62
+
63
+    } catch (err) {
64
+      logger.error('Cannot auto follow hosts of index %s.', indexUrl, { err })
65
+    }
66
+
67
+  }
68
+
69
+  static get Instance () {
70
+    return this.instance || (this.instance = new this())
71
+  }
72
+}

+ 41
- 2
server/models/activitypub/actor-follow.ts View File

@@ -1,5 +1,5 @@
1 1
 import * as Bluebird from 'bluebird'
2
-import { values } from 'lodash'
2
+import { values, difference } from 'lodash'
3 3
 import {
4 4
   AfterCreate,
5 5
   AfterDestroy,
@@ -21,7 +21,7 @@ import { FollowState } from '../../../shared/models/actors'
21 21
 import { ActorFollow } from '../../../shared/models/actors/follow.model'
22 22
 import { logger } from '../../helpers/logger'
23 23
 import { getServerActor } from '../../helpers/utils'
24
-import { ACTOR_FOLLOW_SCORE, FOLLOW_STATES } from '../../initializers/constants'
24
+import { ACTOR_FOLLOW_SCORE, FOLLOW_STATES, SERVER_ACTOR_NAME } from '../../initializers/constants'
25 25
 import { ServerModel } from '../server/server'
26 26
 import { createSafeIn, getSort } from '../utils'
27 27
 import { ActorModel, unusedActorAttributesForAPI } from './actor'
@@ -435,6 +435,45 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
435 435
                            })
436 436
   }
437 437
 
438
+  static async keepUnfollowedInstance (hosts: string[]) {
439
+    const followerId = (await getServerActor()).id
440
+
441
+    const query = {
442
+      attributes: [],
443
+      where: {
444
+        actorId: followerId
445
+      },
446
+      include: [
447
+        {
448
+          attributes: [ ],
449
+          model: ActorModel.unscoped(),
450
+          required: true,
451
+          as: 'ActorFollowing',
452
+          where: {
453
+            preferredUsername: SERVER_ACTOR_NAME
454
+          },
455
+          include: [
456
+            {
457
+              attributes: [ 'host' ],
458
+              model: ServerModel.unscoped(),
459
+              required: true,
460
+              where: {
461
+                host: {
462
+                  [Op.in]: hosts
463
+                }
464
+              }
465
+            }
466
+          ]
467
+        }
468
+      ]
469
+    }
470
+
471
+    const res = await ActorFollowModel.findAll(query)
472
+    const followedHosts = res.map(res => res.ActorFollowing.Server.host)
473
+
474
+    return difference(hosts, followedHosts)
475
+  }
476
+
438 477
   static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) {
439 478
     return ActorFollowModel.createListAcceptedFollowForApiQuery('followers', actorIds, t, start, count)
440 479
   }

+ 71
- 12
server/tests/api/server/auto-follows.ts View File

@@ -6,10 +6,12 @@ import {
6 6
   acceptFollower,
7 7
   cleanupTests,
8 8
   flushAndRunMultipleServers,
9
+  MockInstancesIndex,
9 10
   ServerInfo,
10 11
   setAccessTokensToServers,
11 12
   unfollow,
12
-  updateCustomSubConfig
13
+  updateCustomSubConfig,
14
+  wait
13 15
 } from '../../../../shared/extra-utils/index'
14 16
 import { follow, getFollowersListPaginationAndSort, getFollowingListPaginationAndSort } from '../../../../shared/extra-utils/server/follows'
15 17
 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
@@ -22,13 +24,14 @@ async function checkFollow (follower: ServerInfo, following: ServerInfo, exists:
22 24
     const res = await getFollowersListPaginationAndSort(following.url, 0, 5, '-createdAt')
23 25
     const follows = res.body.data as ActorFollow[]
24 26
 
25
-    if (exists === true) {
26
-      expect(res.body.total).to.equal(1)
27
+    const follow = follows.find(f => {
28
+      return f.follower.host === follower.host && f.state === 'accepted'
29
+    })
27 30
 
28
-      expect(follows[ 0 ].follower.host).to.equal(follower.host)
29
-      expect(follows[ 0 ].state).to.equal('accepted')
31
+    if (exists === true) {
32
+      expect(follow).to.exist
30 33
     } else {
31
-      expect(follows.filter(f => f.state === 'accepted')).to.have.lengthOf(0)
34
+      expect(follow).to.be.undefined
32 35
     }
33 36
   }
34 37
 
@@ -36,13 +39,14 @@ async function checkFollow (follower: ServerInfo, following: ServerInfo, exists:
36 39
     const res = await getFollowingListPaginationAndSort(follower.url, 0, 5, '-createdAt')
37 40
     const follows = res.body.data as ActorFollow[]
38 41
 
39
-    if (exists === true) {
40
-      expect(res.body.total).to.equal(1)
42
+    const follow = follows.find(f => {
43
+      return f.following.host === following.host && f.state === 'accepted'
44
+    })
41 45
 
42
-      expect(follows[ 0 ].following.host).to.equal(following.host)
43
-      expect(follows[ 0 ].state).to.equal('accepted')
46
+    if (exists === true) {
47
+      expect(follow).to.exist
44 48
     } else {
45
-      expect(follows.filter(f => f.state === 'accepted')).to.have.lengthOf(0)
49
+      expect(follow).to.be.undefined
46 50
     }
47 51
   }
48 52
 }
@@ -71,7 +75,7 @@ describe('Test auto follows', function () {
71 75
   before(async function () {
72 76
     this.timeout(30000)
73 77
 
74
-    servers = await flushAndRunMultipleServers(2)
78
+    servers = await flushAndRunMultipleServers(3)
75 79
 
76 80
     // Get the access tokens
77 81
     await setAccessTokensToServers(servers)
@@ -142,6 +146,61 @@ describe('Test auto follows', function () {
142 146
     })
143 147
   })
144 148
 
149
+  describe('Auto follow index', function () {
150
+    const instanceIndexServer = new MockInstancesIndex()
151
+
152
+    before(async () => {
153
+      await instanceIndexServer.initialize()
154
+    })
155
+
156
+    it('Should not auto follow index if the option is not enabled', async function () {
157
+      this.timeout(30000)
158
+
159
+      await wait(5000)
160
+      await waitJobs(servers)
161
+
162
+      await checkFollow(servers[ 0 ], servers[ 1 ], false)
163
+      await checkFollow(servers[ 1 ], servers[ 0 ], false)
164
+    })
165
+
166
+    it('Should auto follow the index', async function () {
167
+      this.timeout(30000)
168
+
169
+      instanceIndexServer.addInstance(servers[1].host)
170
+
171
+      const config = {
172
+        followings: {
173
+          instance: {
174
+            autoFollowIndex: {
175
+              indexUrl: 'http://localhost:42100',
176
+              enabled: true
177
+            }
178
+          }
179
+        }
180
+      }
181
+      await updateCustomSubConfig(servers[0].url, servers[0].accessToken, config)
182
+
183
+      await wait(5000)
184
+      await waitJobs(servers)
185
+
186
+      await checkFollow(servers[ 0 ], servers[ 1 ], true)
187
+
188
+      await resetFollows(servers)
189
+    })
190
+
191
+    it('Should follow new added instances in the index but not old ones', async function () {
192
+      this.timeout(30000)
193
+
194
+      instanceIndexServer.addInstance(servers[2].host)
195
+
196
+      await wait(5000)
197
+      await waitJobs(servers)
198
+
199
+      await checkFollow(servers[ 0 ], servers[ 1 ], false)
200
+      await checkFollow(servers[ 0 ], servers[ 2 ], true)
201
+    })
202
+  })
203
+
145 204
   after(async function () {
146 205
     await cleanupTests(servers)
147 206
   })

+ 1
- 0
shared/extra-utils/index.ts View File

@@ -24,4 +24,5 @@ export * from './videos/video-streaming-playlists'
24 24
 export * from './videos/videos'
25 25
 export * from './videos/video-change-ownership'
26 26
 export * from './feeds/feeds'
27
+export * from './instances-index/mock-instances-index'
27 28
 export * from './search/videos'

+ 38
- 0
shared/extra-utils/instances-index/mock-instances-index.ts View File

@@ -0,0 +1,38 @@
1
+import * as express from 'express'
2
+
3
+export class MockInstancesIndex {
4
+  private indexInstances: { host: string, createdAt: string }[] = []
5
+
6
+  initialize () {
7
+    return new Promise(res => {
8
+      const app = express()
9
+
10
+      app.use('/', (req: express.Request, res: express.Response, next: express.NextFunction) => {
11
+        if (process.env.DEBUG) console.log('Receiving request on mocked server %s.', req.url)
12
+
13
+        return next()
14
+      })
15
+
16
+      app.get('/api/v1/instances/hosts', (req: express.Request, res: express.Response) => {
17
+        const since = req.query.since
18
+
19
+        const filtered = this.indexInstances.filter(i => {
20
+          if (!since) return true
21
+
22
+          return i.createdAt > since
23
+        })
24
+
25
+        return res.json({
26
+          total: filtered.length,
27
+          data: filtered
28
+        })
29
+      })
30
+
31
+      app.listen(42100, () => res())
32
+    })
33
+  }
34
+
35
+  addInstance (host: string) {
36
+    this.indexInstances.push({ host, createdAt: new Date().toISOString() })
37
+  }
38
+}

+ 2
- 1
tsconfig.json View File

@@ -17,7 +17,8 @@
17 17
     "typeRoots": [ "node_modules/@types", "server/typings" ],
18 18
     "baseUrl": "./",
19 19
     "paths": {
20
-      "@server/*": [ "server/*" ]
20
+      "@server/*": [ "server/*" ],
21
+      "@shared/*": [ "shared/*" ]
21 22
     }
22 23
   },
23 24
   "exclude": [

Loading…
Cancel
Save