Merge pull request #117 from mdbenito/fix/117-duplicate-attachments

Fixes for S3 attachments

Author: spruce

README.md

@@ -56,6 +56,10 @@ Go to [Settings / Access Tokens](https://gitlab.com/profile/personal_access_toke
 
 Leave it null for the first run of the script. Then the script will show you which projects there are. Can be either string or number.
 
+#### gitlab.sessionCookie
+
+GitLab's API [does not allow downloading of attachments](https://gitlab.com/gitlab-org/gitlab/-/issues/24155) and only images can be downloaded using HTTP. To work around this limitation and enable binary attachments to be migrated one can use the session cookie set in the browser after logging in to the gitlab instance. The cookie is named `_gitlab_session`.
+
 ### github
 
 #### github.baseUrl

sample_settings.ts

@@ -5,6 +5,7 @@ export default {
     // url: 'https://gitlab.mycompany.com',
     token: '{{gitlab private token}}',
     projectId: null,
+    sessionCookie: null,
   },
   github: {
     // baseUrl: 'https://gitlab.mycompany.com:123/etc',

src/githubHelper.ts

@@ -825,9 +825,7 @@ export default class GithubHelper {
         }
       );
 
-      if (settings.s3) {
-        strWithMigLine = await utils.migrateAttachments(strWithMigLine, this.repoId, settings.s3, this.gitlabHelper);
-      }
+      strWithMigLine = await utils.migrateAttachments(strWithMigLine, this.repoId, settings.s3, this.gitlabHelper);
 
       return strWithMigLine;
     }

src/gitlabHelper.ts

@@ -11,6 +11,7 @@ export default class GitlabHelper {
   gitlabUrl?: string;
   gitlabToken: string;
   gitlabProjectId: number;
+  sessionCookie: string;
 
   host: string;
   projectPath?: string;
@@ -24,6 +25,7 @@ export default class GitlabHelper {
     this.gitlabToken = gitlabSettings.token;
     this.gitlabProjectId = gitlabSettings.projectId;
     this.host = gitlabSettings.url ? gitlabSettings.url : 'http://gitlab.com';
+    this.sessionCookie = gitlabSettings.sessionCookie;
   }
 
   /**
@@ -92,7 +94,13 @@ export default class GitlabHelper {
     try {
       const host = this.host.endsWith('/') ? this.host : this.host + '/';
       const attachmentUrl = host + this.projectPath + relurl;
-      const data = (await axios.get(attachmentUrl, {responseType: 'arraybuffer'})).data;
+      const data = (await axios.get(attachmentUrl, {
+          responseType: 'arraybuffer',
+          headers: {
+            // HACK: work around GitLab's API lack of GET for attachments
+            // See https://gitlab.com/gitlab-org/gitlab/-/issues/24155
+            Cookie: `_gitlab_session=${this.sessionCookie}`
+          }})).data;
       return Buffer.from(data, 'binary')
     } catch (err) {
       console.error(`Could not download attachment #${relurl}.`);

src/settings.ts

@@ -42,6 +42,7 @@ export interface GitlabSettings {
   url?: string;
   token: string;
   projectId: number;
+  sessionCookie: string;
 }
 
 export interface S3Settings {

src/utils.ts

@@ -34,7 +34,7 @@ export const generateUserProjectRegex = () => {
 
 // Creates new attachments and replaces old links
 export const migrateAttachments = async (body: string, githubRepoId: number | undefined, s3: S3Settings, gitlabHelper: GitlabHelper) => {
-  const regexp = /!\[([^\]]+)\]\((\/uploads[^)]+)\)/g;
+  const regexp = /(!?)\[([^\]]+)\]\((\/uploads[^)]+)\)/g;
 
   // Maps link offset to a new name in S3
   const offsetToAttachment: {
@@ -45,45 +45,57 @@ export const migrateAttachments = async (body: string, githubRepoId: number | un
   const matches = body.matchAll(regexp);
 
   for (const match of matches) {
-    const name = match[1];
-    const url = match[2];
-
-    const basename = path.basename(url);
-    const extension = path.extname(url);
-    const mimeType = mime.lookup(basename);
-    const attachmentBuffer = await gitlabHelper.getAttachment(url);
-
-    // Generate new random file name for S3 bucket
-    const id = crypto.randomBytes(16).toString('hex');
-    const newFileName = id + extension;
-    const relativePath = githubRepoId ? `${githubRepoId}/${newFileName}` : newFileName;
-
-    // Doesn't seem like it is easy to upload an issue to github, so upload to S3
-    //https://stackoverflow.com/questions/41581151/how-to-upload-an-image-to-use-in-issue-comments-via-github-api
-
-    const s3url = `https://${s3.bucket}.s3.amazonaws.com/${relativePath}`;
-
-    const s3bucket = new S3();
-    s3bucket.createBucket(() => {
-      const params: S3.PutObjectRequest = {
-        Key: relativePath,
-        Body: attachmentBuffer,
-        ContentType: mimeType === false ? null : mimeType,
-        Bucket: s3.bucket,
-      };
-
-      s3bucket.upload(params, function (err, data) {
-        console.log(`\tUploaded ${basename} to ${s3url}`);
-        if (err) {
-          console.log('ERROR MSG: ', err);
-        }
+    const prefix = match[1] || '';
+    const name = match[2];
+    const url = match[3];
+
+    if (s3 && s3.bucket) {
+      const basename = path.basename(url);
+      const mimeType = mime.lookup(basename);
+      const attachmentBuffer = await gitlabHelper.getAttachment(url);
+      if (!attachmentBuffer) {
+        continue;
+      }
+
+      // // Generate file name for S3 bucket from URL
+      const hash = crypto.createHash('sha256');
+      hash.update(url);
+      const newFileName = hash.digest('hex') + '/' + basename
+      const relativePath = githubRepoId ? `${githubRepoId}/${newFileName}` : newFileName;
+      // Doesn't seem like it is easy to upload an issue to github, so upload to S3
+      //https://stackoverflow.com/questions/41581151/how-to-upload-an-image-to-use-in-issue-comments-via-github-api
+
+      const s3url = `https://${s3.bucket}.s3.amazonaws.com/${relativePath}`;
+
+      const s3bucket = new S3();
+      s3bucket.createBucket(() => {
+        const params: S3.PutObjectRequest = {
+          Key: relativePath,
+          Body: attachmentBuffer,
+          ContentType: mimeType === false ? null : mimeType,
+          Bucket: s3.bucket,
+        };
+
+        s3bucket.upload(params, function (err, data) {
+          console.log(`\tUploading ${basename} to ${s3url}... `);
+          if (err) {
+            console.log('ERROR: ', err);
+          } else {
+            console.log(`\t...Done uploading`);
+          }
+        });
       });
-    });
-
-    // Add the new URL to the map
-    offsetToAttachment[match.index] = `![${name}](${s3url})`;
-
+    
+      // Add the new URL to the map
+      offsetToAttachment[match.index] = `${prefix}[${name}](${s3url})`;
+    } else {
+      // Not using S3: default to old URL, adding absolute path
+      const host = gitlabHelper.host.endsWith('/') 
+                    ? gitlabHelper.host : gitlabHelper.host + '/';
+      const attachmentUrl = host + gitlabHelper.projectPath + url;
+      offsetToAttachment[match.index] = `${prefix}[${name}](${attachmentUrl})`;
+    }
   }
 
-  return body.replace(regexp, ({},{},{},offset,{}) => offsetToAttachment[offset]);
+  return body.replace(regexp, ({},{},{},{},offset,{}) => offsetToAttachment[offset]);
 };