Browse Source

Add gitlab ci support

Chocobozzz 3 months ago
parent
commit
2284f20207
No account linked to committer's email address

+ 100
- 0
.gitlab-ci.yml View File

@@ -0,0 +1,100 @@
1
+image: chocobozzz/peertube-ci:10
2
+
3
+stages:
4
+  - build-and-lint
5
+  - test
6
+  - nightly
7
+
8
+before_script:
9
+  - 'sed -i -z "s/database:\n  hostname: ''localhost''/database:\n  hostname: ''postgres''/" config/test.yaml'
10
+  - 'sed -i -z "s/redis:\n  hostname: ''localhost''/redis:\n  hostname: ''redis''/" config/test.yaml'
11
+  - if [[ $CI_JOB_STAGE == "test" ]]; then psql -c "create user peertube with password 'peertube';"; fi
12
+  - NOCLIENT=1 yarn install --pure-lockfile --cache-folder .yarn-cache
13
+
14
+cache:
15
+  key: yarn
16
+  paths:
17
+    - .yarn-cache
18
+    - cached-fixtures
19
+
20
+###
21
+## Jobs templates
22
+#
23
+#.build-and-lint: &build-and-lint
24
+#  stage: build-and-lint
25
+#
26
+#.tests: &tests
27
+#  stage: test
28
+#  dependencies:
29
+#    - build-server
30
+#  services:
31
+#    - name: postgres:9.6
32
+#      alias: postgres
33
+#    - name: redis:latest
34
+#      alias: redis
35
+#  variables:
36
+#    PGHOST: postgres
37
+#    PGUSER: postgres
38
+#    REDIS_HOST: redis
39
+#  artifacts:
40
+#    expire_in: 1 day
41
+#    paths:
42
+#      - test*/logs
43
+#    when: always
44
+#
45
+####
46
+### Build and lint
47
+##
48
+#build-server:
49
+#  <<: *build-and-lint
50
+#  artifacts:
51
+#    expire_in: 5h
52
+#    paths:
53
+#      - dist/
54
+#  script:
55
+#    - npm run build:server
56
+#
57
+#lint:
58
+#  <<: *build-and-lint
59
+#  script:
60
+#    - yarn install --pure-lockfile --cache-folder .yarn-cache
61
+#    - npm run ci -- "lint"
62
+#
63
+####
64
+### Tests
65
+#
66
+#test-misc:
67
+#  <<: *tests
68
+#  script:
69
+#    - yarn install --pure-lockfile --cache-folder .yarn-cache
70
+#    - npm run ci -- "misc"
71
+#
72
+#test-cli:
73
+#  <<: *tests
74
+#  retry:
75
+#    max: 1
76
+#  script:
77
+#    - npm run ci -- "cli"
78
+#
79
+#api:
80
+#  <<: *tests
81
+#  parallel: 4
82
+#  retry:
83
+#    max: 1
84
+#  script:
85
+#    - NODE_PENDING_JOB_WAIT=1000 npm run ci -- api-$CI_NODE_INDEX
86
+
87
+build-nightly:
88
+  stage: nightly
89
+  only:
90
+    - schedules
91
+  script:
92
+    - yarn install --pure-lockfile --cache-folder .yarn-cache
93
+    - npm run nightly
94
+    - mkdir "${HOME}/.ssh"
95
+    - chmod 700 "${HOME}/.ssh"
96
+    - if [ ! -z ${DEPLOYEMENT_KNOWN_HOSTS+x} ]; then echo -e "${DEPLOYEMENT_KNOWN_HOSTS}" > ${HOME}/.ssh/known_hosts; fi
97
+    - eval `ssh-agent -s`
98
+    - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi
99
+    - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi
100
+

+ 1
- 1
.travis.yml View File

@@ -47,7 +47,7 @@ matrix:
47 47
   - env: TEST_SUITE=lint
48 48
 
49 49
 script:
50
-  - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run travis -- "$TEST_SUITE"
50
+  - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run ci -- "$TEST_SUITE"
51 51
 
52 52
 after_failure:
53 53
   - cat test1/logs/peertube.log

+ 2
- 2
README.md View File

@@ -45,8 +45,8 @@ Be part of a network of multiple small federated, interoperable video hosting pr
45 45
 
46 46
   <br />
47 47
 
48
-  <a href="https://travis-ci.org/Chocobozzz/PeerTube">
49
-    <img src="https://travis-ci.org/Chocobozzz/PeerTube.svg?branch=develop" alt="Build Status" />
48
+  <a href="https://framagit.org/framasoft/peertube/PeerTube/commits/develop">
49
+    <img alt="pipeline status" src="https://framagit.org/framasoft/peertube/PeerTube/badges/develop/pipeline.svg" />
50 50
   </a>
51 51
 
52 52
   <a href="https://david-dm.org/Chocobozzz/PeerTube">

+ 3
- 0
config/test.yaml View File

@@ -20,6 +20,9 @@ database:
20 20
   hostname: 'localhost'
21 21
   port: 5432
22 22
 
23
+redis:
24
+  hostname: 'localhost'
25
+
23 26
 smtp:
24 27
   hostname: null
25 28
   port: 1025

+ 6
- 4
package.json View File

@@ -60,17 +60,19 @@
60 60
     "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)",
61 61
     "tsc": "tsc",
62 62
     "commander": "commander",
63
-    "lint": "npm run travis -- lint",
63
+    "lint": "npm run ci -- lint",
64 64
     "ng": "ng",
65 65
     "nodemon": "nodemon",
66 66
     "ts-node": "ts-node",
67 67
     "tslint": "tslint",
68 68
     "concurrently": "concurrently",
69
+    "mocha-parallel-tests": "mocha-parallel-tests",
69 70
     "sasslint": "sass-lint --verbose --no-exit",
70 71
     "sasslint:fix": "sass-lint-auto-fix -c .sass-lint.yml --verbose",
71 72
     "mocha": "mocha",
72
-    "travis": "scripty",
73
+    "ci": "scripty",
73 74
     "release": "scripty",
75
+    "nightly": "scripty",
74 76
     "client-report": "scripty"
75 77
   },
76 78
   "husky": {
@@ -93,7 +95,7 @@
93 95
     ]
94 96
   },
95 97
   "resolutions": {
96
-    "@types/bluebird": "3.5.21"
98
+    "@types/bluebird": "3.5.27"
97 99
   },
98 100
   "dependencies": {
99 101
     "apicache": "^1.4.0",
@@ -206,7 +208,7 @@
206 208
     "maildev": "^1.0.0-rc3",
207 209
     "marked-man": "^0.6.0",
208 210
     "mocha": "^6.0.0",
209
-    "mocha-parallel-tests": "^2.1.0",
211
+    "mocha-parallel-tests": "^2.2.1",
210 212
     "nodemon": "^1.18.6",
211 213
     "sass-lint": "^1.12.1",
212 214
     "source-map-support": "^0.5.0",

scripts/travis.sh → scripts/ci.sh View File

@@ -22,16 +22,16 @@ elif [ "$1" = "cli" ]; then
22 22
     mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/cli/index.ts
23 23
 elif [ "$1" = "api-1" ]; then
24 24
     npm run build:server
25
-    sh ./server/tests/api/travis-1.sh 2
25
+    sh ./server/tests/api/ci-1.sh 2
26 26
 elif [ "$1" = "api-2" ]; then
27 27
     npm run build:server
28
-    sh ./server/tests/api/travis-2.sh 2
28
+    sh ./server/tests/api/ci-2.sh 2
29 29
 elif [ "$1" = "api-3" ]; then
30 30
     npm run build:server
31
-    sh ./server/tests/api/travis-3.sh 2
31
+    sh ./server/tests/api/ci-3.sh 2
32 32
 elif [ "$1" = "api-4" ]; then
33 33
     npm run build:server
34
-    sh ./server/tests/api/travis-4.sh 2
34
+    sh ./server/tests/api/ci-4.sh 2
35 35
 elif [ "$1" = "lint" ]; then
36 36
     npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts"
37 37
 

+ 9
- 4
scripts/clean/server/test.sh View File

@@ -5,7 +5,7 @@ set -eu
5 5
 recreateDB () {
6 6
   dbname="peertube_test$1"
7 7
 
8
-  dropdb --if-exists "$dbname"
8
+  dropdb --if-exists "$dbname" 2>&1
9 9
 
10 10
   createdb -O peertube "$dbname"
11 11
   psql -c "CREATE EXTENSION pg_trgm;" "$dbname" &
@@ -18,10 +18,15 @@ removeFiles () {
18 18
 
19 19
 dropRedis () {
20 20
   port=$((9000+$1))
21
+  host="localhost"
21 22
 
22
-  redis-cli KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
23
-  redis-cli KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
24
-  redis-cli KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
23
+  if [ ! -z ${GITLAB_CI+x} ]; then
24
+    host="redis"
25
+  fi
26
+
27
+  redis-cli -h "$host" KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
28
+  redis-cli -h "$host" KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
29
+  redis-cli -h "$host" KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
25 30
 }
26 31
 
27 32
 seq=$(seq 1 6)

+ 42
- 0
scripts/nightly.sh View File

@@ -0,0 +1,42 @@
1
+#!/bin/bash
2
+
3
+set -eu
4
+
5
+shutdown() {
6
+  # Get our process group id
7
+  # shellcheck disable=SC2009
8
+  PGID=$(ps -o pgid= $$ | grep -o "[0-9]*")
9
+
10
+  # Kill it in a new new process group
11
+  setsid kill -- -"$PGID"
12
+  exit 0
13
+}
14
+
15
+trap "shutdown" SIGINT SIGTERM
16
+
17
+today=$(date '+%F')
18
+directory_name="peertube-nightly-$today"
19
+tar_name="peertube-nightly-$today.tar.xz"
20
+
21
+npm run build
22
+
23
+# Creating the archives
24
+(
25
+  # local variables
26
+  directories_to_archive=("$directory_name/CREDITS.md" "$directory_name/FAQ.md" \
27
+                          "$directory_name/LICENSE" "$directory_name/README.md" \
28
+                          "$directory_name/client/dist/" "$directory_name/client/yarn.lock" \
29
+                          "$directory_name/client/package.json" "$directory_name/config" \
30
+                          "$directory_name/dist" "$directory_name/package.json" \
31
+                          "$directory_name/scripts" "$directory_name/support" \
32
+                          "$directory_name/tsconfig.json" "$directory_name/yarn.lock")
33
+
34
+  # temporary setup
35
+  cd ..
36
+  ln -s "PeerTube" "$directory_name"
37
+
38
+  XZ_OPT=-e9 tar cfJ "PeerTube/$tar_name" "${directories_to_archive[@]}"
39
+
40
+  # temporary setup destruction
41
+  rm "$directory_name"
42
+)

+ 1
- 1
scripts/test.sh View File

@@ -5,6 +5,6 @@ set -eu
5 5
 npm run build:server
6 6
 npm run setup:cli
7 7
 
8
-npm run travis -- lint
8
+npm run ci -- lint
9 9
 
10 10
 mocha --exit --require ts-node/register/type-check --bail server/tests/index.ts

+ 3
- 3
server/controllers/api/videos/abuse.ts View File

@@ -113,16 +113,16 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
113 113
 
114 114
     // We send the video abuse to the origin server
115 115
     if (videoInstance.isOwned() === false) {
116
-      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
116
+      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
117 117
     }
118 118
 
119
-    Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
120
-
121 119
     auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
122 120
 
123 121
     return videoAbuseInstance
124 122
   })
125 123
 
124
+  Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
125
+
126 126
   logger.info('Abuse report for video %s created.', videoInstance.name)
127 127
 
128 128
   return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end()

+ 9
- 1
server/helpers/database-utils.ts View File

@@ -2,6 +2,7 @@ import * as retry from 'async/retry'
2 2
 import * as Bluebird from 'bluebird'
3 3
 import { Model } from 'sequelize-typescript'
4 4
 import { logger } from './logger'
5
+import { Transaction } from 'sequelize'
5 6
 
6 7
 function retryTransactionWrapper <T, A, B, C> (
7 8
   functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T> | Bluebird<T>,
@@ -72,11 +73,18 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
72 73
   })
73 74
 }
74 75
 
76
+function afterCommitIfTransaction (t: Transaction, fn: Function) {
77
+  if (t) return t.afterCommit(() => fn())
78
+
79
+  return fn()
80
+}
81
+
75 82
 // ---------------------------------------------------------------------------
76 83
 
77 84
 export {
78 85
   resetSequelizeInstance,
79 86
   retryTransactionWrapper,
80 87
   transactionRetryer,
81
-  updateInstanceWithAnother
88
+  updateInstanceWithAnother,
89
+  afterCommitIfTransaction
82 90
 }

+ 5
- 3
server/lib/activitypub/process/process-flag.ts View File

@@ -31,7 +31,7 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
31 31
 
32 32
   const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object })
33 33
 
34
-  return sequelizeTypescript.transaction(async t => {
34
+  const videoAbuse = await sequelizeTypescript.transaction(async t => {
35 35
     const videoAbuseData = {
36 36
       reporterAccountId: account.id,
37 37
       reason: flag.content,
@@ -42,8 +42,10 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
42 42
     const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t })
43 43
     videoAbuseInstance.Video = video
44 44
 
45
-    Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
46
-
47 45
     logger.info('Remote abuse for video uuid %s created', flag.object)
46
+
47
+    return videoAbuseInstance
48 48
   })
49
+
50
+  Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
49 51
 }

+ 3
- 2
server/lib/activitypub/send/send-create.ts View File

@@ -11,6 +11,7 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc
11 11
 import { VideoPlaylistModel } from '../../../models/video/video-playlist'
12 12
 import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
13 13
 import { getServerActor } from '../../../helpers/utils'
14
+import * as Bluebird from 'bluebird'
14 15
 
15 16
 async function sendCreateVideo (video: VideoModel, t: Transaction) {
16 17
   if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -82,7 +83,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
82 83
 
83 84
   // This was a reply, send it to the parent actors
84 85
   const actorsException = [ byActor ]
85
-  await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
86
+  await broadcastToActors(createActivity, byActor, parentsCommentActors, t, actorsException)
86 87
 
87 88
   // Broadcast to our followers
88 89
   await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
@@ -91,7 +92,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
91 92
   if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
92 93
 
93 94
   // Send to origin
94
-  return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
95
+  t.afterCommit(() => unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
95 96
 }
96 97
 
97 98
 function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {

+ 2
- 2
server/lib/activitypub/send/send-delete.ts View File

@@ -59,7 +59,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
59 59
 
60 60
   // This was a reply, send it to the parent actors
61 61
   const actorsException = [ byActor ]
62
-  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
62
+  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), t, actorsException)
63 63
 
64 64
   // Broadcast to our followers
65 65
   await broadcastToFollowers(activity, byActor, [ byActor ], t)
@@ -68,7 +68,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
68 68
   if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
69 69
 
70 70
   // Send to origin
71
-  return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
71
+  t.afterCommit(() => unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
72 72
 }
73 73
 
74 74
 async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {

+ 3
- 2
server/lib/activitypub/send/send-flag.ts View File

@@ -6,8 +6,9 @@ import { unicastTo } from './utils'
6 6
 import { logger } from '../../../helpers/logger'
7 7
 import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub'
8 8
 import { audiencify, getAudience } from '../audience'
9
+import { Transaction } from 'sequelize'
9 10
 
10
-async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
11
+async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) {
11 12
   if (!video.VideoChannel.Account.Actor.serverId) return // Local user
12 13
 
13 14
   const url = getVideoAbuseActivityPubUrl(videoAbuse)
@@ -18,7 +19,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
18 19
   const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
19 20
   const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience)
20 21
 
21
-  return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
22
+  t.afterCommit(() => unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl))
22 23
 }
23 24
 
24 25
 function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag {

+ 3
- 2
server/lib/activitypub/send/send-follow.ts View File

@@ -4,8 +4,9 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
4 4
 import { getActorFollowActivityPubUrl } from '../url'
5 5
 import { unicastTo } from './utils'
6 6
 import { logger } from '../../../helpers/logger'
7
+import { Transaction } from 'sequelize'
7 8
 
8
-function sendFollow (actorFollow: ActorFollowModel) {
9
+function sendFollow (actorFollow: ActorFollowModel, t: Transaction) {
9 10
   const me = actorFollow.ActorFollower
10 11
   const following = actorFollow.ActorFollowing
11 12
 
@@ -17,7 +18,7 @@ function sendFollow (actorFollow: ActorFollowModel) {
17 18
   const url = getActorFollowActivityPubUrl(me, following)
18 19
   const data = buildFollowActivity(url, me, following)
19 20
 
20
-  return unicastTo(data, me, following.inboxUrl)
21
+  t.afterCommit(() => unicastTo(data, me, following.inboxUrl))
21 22
 }
22 23
 
23 24
 function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {

+ 1
- 1
server/lib/activitypub/send/send-undo.ts View File

@@ -37,7 +37,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
37 37
   const followActivity = buildFollowActivity(followUrl, me, following)
38 38
   const undoActivity = undoActivityData(undoUrl, me, followActivity)
39 39
 
40
-  return unicastTo(undoActivity, me, following.inboxUrl)
40
+  t.afterCommit(() => unicastTo(undoActivity, me, following.inboxUrl))
41 41
 }
42 42
 
43 43
 async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {

+ 3
- 1
server/lib/activitypub/send/send-update.ts View File

@@ -26,7 +26,9 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct
26 26
   const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString())
27 27
 
28 28
   // Needed to build the AP object
29
-  if (!video.VideoCaptions) video.VideoCaptions = await video.$get('VideoCaptions') as VideoCaptionModel[]
29
+  if (!video.VideoCaptions) {
30
+    video.VideoCaptions = await video.$get('VideoCaptions', { transaction: t }) as VideoCaptionModel[]
31
+  }
30 32
 
31 33
   const videoObject = video.toActivityPubObject()
32 34
   const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)

+ 13
- 7
server/lib/activitypub/send/utils.ts View File

@@ -7,6 +7,7 @@ import { JobQueue } from '../../job-queue'
7 7
 import { VideoModel } from '../../../models/video/video'
8 8
 import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
9 9
 import { getServerActor } from '../../../helpers/utils'
10
+import { afterCommitIfTransaction } from '../../../helpers/database-utils'
10 11
 
11 12
 async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
12 13
   byActor: ActorModel,
@@ -20,7 +21,9 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
20 21
     const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo)
21 22
     const activity = activityBuilder(audience)
22 23
 
23
-    return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
24
+    return afterCommitIfTransaction(options.transaction, () => {
25
+      return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
26
+    })
24 27
   }
25 28
 
26 29
   // Send to followers
@@ -28,6 +31,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
28 31
   const activity = activityBuilder(audience)
29 32
 
30 33
   const actorsException = [ options.byActor ]
34
+
31 35
   return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException)
32 36
 }
33 37
 
@@ -76,7 +80,7 @@ async function forwardActivity (
76 80
     uris,
77 81
     body: activity
78 82
   }
79
-  return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })
83
+  return afterCommitIfTransaction(t, () => JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }))
80 84
 }
81 85
 
82 86
 async function broadcastToFollowers (
@@ -87,20 +91,22 @@ async function broadcastToFollowers (
87 91
   actorsException: ActorModel[] = []
88 92
 ) {
89 93
   const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
90
-  return broadcastTo(uris, data, byActor)
94
+
95
+  return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
91 96
 }
92 97
 
93 98
 async function broadcastToActors (
94 99
   data: any,
95 100
   byActor: ActorModel,
96 101
   toActors: ActorModel[],
102
+  t?: Transaction,
97 103
   actorsException: ActorModel[] = []
98 104
 ) {
99 105
   const uris = await computeUris(toActors, actorsException)
100
-  return broadcastTo(uris, data, byActor)
106
+  return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
101 107
 }
102 108
 
103
-async function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
109
+function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
104 110
   if (uris.length === 0) return undefined
105 111
 
106 112
   logger.debug('Creating broadcast job.', { uris })
@@ -114,7 +120,7 @@ async function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
114 120
   return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })
115 121
 }
116 122
 
117
-async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
123
+function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
118 124
   logger.debug('Creating unicast job.', { uri: toActorUrl })
119 125
 
120 126
   const payload = {
@@ -123,7 +129,7 @@ async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
123 129
     body: data
124 130
   }
125 131
 
126
-  return JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload })
132
+  JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload })
127 133
 }
128 134
 
129 135
 // ---------------------------------------------------------------------------

+ 1
- 1
server/lib/job-queue/handlers/activitypub-follow.ts View File

@@ -69,7 +69,7 @@ async function follow (fromActor: ActorModel, targetActor: ActorModel) {
69 69
     actorFollow.ActorFollower = fromActor
70 70
 
71 71
     // Send a notification to remote server if our follow is not already accepted
72
-    if (actorFollow.state !== 'accepted') await sendFollow(actorFollow)
72
+    if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t)
73 73
 
74 74
     return actorFollow
75 75
   })

+ 1
- 1
server/tests/api/activitypub/fetch.ts View File

@@ -87,7 +87,7 @@ describe('Test ActivityPub fetcher', function () {
87 87
   })
88 88
 
89 89
   after(async function () {
90
-    this.timeout(10000)
90
+    this.timeout(20000)
91 91
 
92 92
     await cleanupTests(servers)
93 93
 

server/tests/api/travis-1.sh → server/tests/api/ci-1.sh View File

@@ -6,5 +6,5 @@ checkParamFiles=$(find server/tests/api/check-params -type f | grep -v index.ts
6 6
 notificationsFiles=$(find server/tests/api/notifications -type f | grep -v index.ts | xargs echo)
7 7
 searchFiles=$(find server/tests/api/search -type f | grep -v index.ts | xargs echo)
8 8
 
9
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
9
+MOCHA_PARALLEL=true npm run mocha --timeout 30000 --exit --require ts-node/register --bail \
10 10
     $notificationsFiles $searchFiles $checkParamFiles

server/tests/api/travis-2.sh → server/tests/api/ci-2.sh View File

@@ -5,5 +5,5 @@ set -eu
5 5
 serverFiles=$(find server/tests/api/server -type f | grep -v index.ts | xargs echo)
6 6
 usersFiles=$(find server/tests/api/users -type f | grep -v index.ts | xargs echo)
7 7
 
8
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
8
+MOCHA_PARALLEL=true npm run mocha-parallel-tests -- --max-parallel $1 --timeout 30000 --exit --require ts-node/register --bail \
9 9
     $serverFiles $usersFiles

server/tests/api/travis-3.sh → server/tests/api/ci-3.sh View File

@@ -4,5 +4,5 @@ set -eu
4 4
 
5 5
 videosFiles=$(find server/tests/api/videos -type f | grep -v index.ts | xargs echo)
6 6
 
7
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
7
+MOCHA_PARALLEL=true npm run mocha --timeout 30000 --exit --require ts-node/register --bail \
8 8
     $videosFiles

server/tests/api/travis-4.sh → server/tests/api/ci-4.sh View File

@@ -5,5 +5,5 @@ set -eu
5 5
 redundancyFiles=$(find server/tests/api/redundancy -type f | grep -v index.ts | xargs echo)
6 6
 activitypubFiles=$(find server/tests/api/activitypub -type f | grep -v index.ts | xargs echo)
7 7
 
8
-MOCHA_PARALLEL=true mocha-parallel-tests --max-parallel $1 --timeout 5000 --exit --require ts-node/register --bail \
8
+MOCHA_PARALLEL=true npm run mocha-parallel-tests -- --max-parallel $1 --timeout 30000 --exit --require ts-node/register --bail \
9 9
     $redundancyFiles $activitypubFiles

+ 2
- 2
server/tests/api/notifications/user-notifications.ts View File

@@ -782,7 +782,7 @@ describe('Test users notifications', function () {
782 782
     it('Should not send a notification before the video is published', async function () {
783 783
       this.timeout(20000)
784 784
 
785
-      let updateAt = new Date(new Date().getTime() + 100000)
785
+      let updateAt = new Date(new Date().getTime() + 1000000)
786 786
 
787 787
       const data = {
788 788
         privacy: VideoPrivacy.PRIVATE,
@@ -1074,7 +1074,7 @@ describe('Test users notifications', function () {
1074 1074
     it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () {
1075 1075
       this.timeout(20000)
1076 1076
 
1077
-      let updateAt = new Date(new Date().getTime() + 100000)
1077
+      let updateAt = new Date(new Date().getTime() + 1000000)
1078 1078
 
1079 1079
       const name = 'video with auto-blacklist and future schedule ' + uuidv4()
1080 1080
 

+ 1
- 1
server/tests/api/server/email.ts View File

@@ -40,7 +40,7 @@ describe('Test emails', function () {
40 40
   let emailPort: number
41 41
 
42 42
   before(async function () {
43
-    this.timeout(30000)
43
+    this.timeout(50000)
44 44
 
45 45
     emailPort = await MockSmtpServer.Instance.collectEmails(emails)
46 46
 

+ 1
- 1
server/tests/api/server/handle-down.ts View File

@@ -109,7 +109,7 @@ describe('Test handle downs', function () {
109 109
   })
110 110
 
111 111
   it('Should remove followers that are often down', async function () {
112
-    this.timeout(60000)
112
+    this.timeout(240000)
113 113
 
114 114
     // Server 2 and 3 follow server 1
115 115
     await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)

+ 2
- 1
server/tests/api/server/jobs.ts View File

@@ -8,6 +8,7 @@ import { getJobsList, getJobsListPaginationAndSort, waitJobs } from '../../../..
8 8
 import { flushAndRunMultipleServers } from '../../../../shared/extra-utils/server/servers'
9 9
 import { uploadVideo } from '../../../../shared/extra-utils/videos/videos'
10 10
 import { dateIsValid } from '../../../../shared/extra-utils/miscs/miscs'
11
+import { Job } from '../../../../shared/models/server'
11 12
 
12 13
 const expect = chai.expect
13 14
 
@@ -50,7 +51,7 @@ describe('Test jobs', function () {
50 51
     if (job.type === 'videos-views') job = res.body.data[1]
51 52
 
52 53
     expect(job.state).to.equal('completed')
53
-    expect(job.type).to.equal('activitypub-follow')
54
+    expect(job.type.startsWith('activitypub-')).to.be.true
54 55
     expect(dateIsValid(job.createdAt)).to.be.true
55 56
     expect(dateIsValid(job.processedOn)).to.be.true
56 57
     expect(dateIsValid(job.finishedOn)).to.be.true

+ 1
- 1
server/tests/api/videos/video-schedule-update.ts View File

@@ -85,7 +85,7 @@ describe('Test video update scheduler', function () {
85 85
   })
86 86
 
87 87
   it('Should wait some seconds and have the video in public privacy', async function () {
88
-    this.timeout(20000)
88
+    this.timeout(50000)
89 89
 
90 90
     await wait(15000)
91 91
     await waitJobs(servers)

+ 6
- 4
shared/extra-utils/miscs/miscs.ts View File

@@ -1,10 +1,10 @@
1 1
 /* tslint:disable:no-unused-expression */
2 2
 
3 3
 import * as chai from 'chai'
4
-import { basename, isAbsolute, join, resolve } from 'path'
4
+import { basename, dirname, isAbsolute, join, resolve } from 'path'
5 5
 import * as request from 'supertest'
6 6
 import * as WebTorrent from 'webtorrent'
7
-import { pathExists, readFile } from 'fs-extra'
7
+import { ensureDir, pathExists, readFile } from 'fs-extra'
8 8
 import * as ffmpeg from 'fluent-ffmpeg'
9 9
 
10 10
 const expect = chai.expect
@@ -59,12 +59,12 @@ async function testImage (url: string, imageName: string, imagePath: string, ext
59 59
   expect(data.length).to.be.below(maxLength)
60 60
 }
61 61
 
62
-function buildAbsoluteFixturePath (path: string, customTravisPath = false) {
62
+function buildAbsoluteFixturePath (path: string, customCIPath = false) {
63 63
   if (isAbsolute(path)) {
64 64
     return path
65 65
   }
66 66
 
67
-  if (customTravisPath && process.env.TRAVIS) return join(process.env.HOME, 'fixtures', path)
67
+  if (customCIPath && process.env.GITLAB_CI) return join(root(), 'cached-fixtures', path)
68 68
 
69 69
   return join(root(), 'server', 'tests', 'fixtures', path)
70 70
 }
@@ -72,6 +72,8 @@ function buildAbsoluteFixturePath (path: string, customTravisPath = false) {
72 72
 async function generateHighBitrateVideo () {
73 73
   const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true)
74 74
 
75
+  await ensureDir(dirname(tempFixturePath))
76
+
75 77
   const exists = await pathExists(tempFixturePath)
76 78
   if (!exists) {
77 79
 

+ 1
- 2
shared/extra-utils/miscs/sql.ts View File

@@ -1,6 +1,5 @@
1 1
 import { QueryTypes, Sequelize } from 'sequelize'
2 2
 import { ServerInfo } from '../server/servers'
3
-import { PluginType } from '../../models/plugins/plugin.type'
4 3
 
5 4
 let sequelizes: { [ id: number ]: Sequelize } = {}
6 5
 
@@ -10,7 +9,7 @@ function getSequelize (internalServerNumber: number) {
10 9
   const dbname = 'peertube_test' + internalServerNumber
11 10
   const username = 'peertube'
12 11
   const password = 'peertube'
13
-  const host = 'localhost'
12
+  const host = process.env.GITLAB_CI ? 'postgres' : 'localhost'
14 13
   const port = 5432
15 14
 
16 15
   const seq = new Sequelize(dbname, username, password, {

+ 4
- 1
shared/extra-utils/server/jobs.ts View File

@@ -2,6 +2,7 @@ import * as request from 'supertest'
2 2
 import { Job, JobState } from '../../models'
3 3
 import { wait } from '../miscs/miscs'
4 4
 import { ServerInfo } from './servers'
5
+import { inspect } from 'util'
5 6
 
6 7
 function getJobsList (url: string, accessToken: string, state: JobState) {
7 8
   const path = '/api/v1/jobs/' + state
@@ -49,7 +50,9 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
49 50
           .then(res => res.body.data)
50 51
           .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views'))
51 52
           .then(jobs => {
52
-            if (jobs.length !== 0) pendingRequests = true
53
+            if (jobs.length !== 0) {
54
+              pendingRequests = true
55
+            }
53 56
           })
54 57
         tasks.push(p)
55 58
       }

+ 2
- 2
shared/extra-utils/server/servers.ts View File

@@ -79,8 +79,8 @@ function flushTests (serverNumber?: number) {
79 79
   return new Promise<void>((res, rej) => {
80 80
     const suffix = serverNumber ? ` -- ${serverNumber}` : ''
81 81
 
82
-    return exec('npm run clean:server:test' + suffix, err => {
83
-      if (err) return rej(err)
82
+    return exec('npm run clean:server:test' + suffix, (err, _stdout, stderr) => {
83
+      if (err || stderr) return rej(err || new Error(stderr))
84 84
 
85 85
       return res()
86 86
     })

+ 1
- 1
support/doc/development/release.md View File

@@ -10,7 +10,7 @@
10 10
     * `NODE_APP_INSTANCE=6 NODE_ENV=test npm run start` and check migrations does not fail
11 11
  * Run `rm -r node_modules && rm -r client/node_modules && yarn install --pure-lockfile && npm run build` to see if all the supported languages compile correctly
12 12
  * Update https://peertube2.cpy.re and check it works correctly
13
- * Check Travis tests are green
13
+ * Check CI tests are green
14 14
  * Run E2E tests: `BROWSERSTACK_USER=my_user BROWSERSTACK_KEY=my_key npm run e2e`
15 15
  * Release: `GITHUB_TOKEN=my_token npm run release -- 1.x.x`
16 16
  * Create a dedicated branch: `git checkout -b release/1.x.x && git push origin release/1.x.x`

+ 7
- 2
yarn.lock View File

@@ -54,7 +54,12 @@
54 54
   dependencies:
55 55
     "@types/node" "*"
56 56
 
57
-"@types/bluebird@*", "@types/bluebird@3.5.21":
57
+"@types/bluebird@*", "@types/bluebird@3.5.27":
58
+  version "3.5.27"
59
+  resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.27.tgz#61eb4d75dc6bfbce51cf49ee9bbebe941b2cb5d0"
60
+  integrity sha512-6BmYWSBea18+tSjjSC3QIyV93ZKAeNWGM7R6aYt1ryTZXrlHF+QLV0G2yV0viEGVyRkyQsWfMoJ0k/YghBX5sQ==
61
+
62
+"@types/bluebird@3.5.21":
58 63
   version "3.5.21"
59 64
   resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.21.tgz#567615589cc913e84a28ecf9edb031732bdf2634"
60 65
   integrity sha512-6UNEwyw+6SGMC/WMI0ld0PS4st7Qq51qgguFrFizOSpGvZiqe9iswztFSdZvwJBEhLOy2JaxNE6VC7yMAlbfyQ==
@@ -4678,7 +4683,7 @@ mkdirp@0.5.1, mkdirp@0.x.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
4678 4683
   dependencies:
4679 4684
     minimist "0.0.8"
4680 4685
 
4681
-mocha-parallel-tests@^2.1.0:
4686
+mocha-parallel-tests@^2.2.1:
4682 4687
   version "2.2.1"
4683 4688
   resolved "https://registry.yarnpkg.com/mocha-parallel-tests/-/mocha-parallel-tests-2.2.1.tgz#616b316c9f20c956a6ce18cd38d45e7db3045b99"
4684 4689
   integrity sha512-9t0NtQ2V/KrNKaUj08RVZp0dqJxeRMTGvC7JA5Daw/LA8GOP+4P7NxRgU0e2sM0JRWYxSakXtpE+MEVGX1ifuw==

Loading…
Cancel
Save