Move to on-the-fly parsing

Author: AbrilRBS

app.js

@@ -342,9 +342,10 @@ aws.initConfig(awsProps)
                                 ogTitle: result.specialMetadata ? result.specialMetadata.title.S : "Compiler Explorer"
                             };
                             if (!metadata.ogDescription) {
-                                if (result.metadata && result.metadata.single_code && result.metadata.single_code.M) {
-                                    const lang = languages[result.metadata.single_code.M.language.S];
-                                    metadata.ogDescription = result.metadata.single_code.M.source.S;
+                                const sources = utils.glGetEditorSources(config.content);
+                                if (sources.length === 1) {
+                                    const lang = languages[sources[0].language];
+                                    metadata.ogDescription = sources[0].source;
                                     if (lang) {
                                         metadata.ogTitle += ` - ${lang.name}`;
                                     }

lib/storage/storage-s3.js

@@ -47,20 +47,6 @@ class StorageS3 extends StorageBase {
         this.dynamoDb = new AWS.DynamoDB();
     }
 
-    getSources(content) {
-        let sources = [];
-        _.each(content, element => {
-            if (element.type === 'component') {
-                if (element.componentName === 'codeEditor') {
-                    sources.push({source: element.componentState.source, language: element.componentState.lang});
-                }
-            } else {
-                sources = this.getSources(element.content).concat(sources);
-            }
-        });
-        return sources;
-    }
-
     storeItem(item, req) {
         logger.info(`Storing item ${item.prefix}`);
         const now = new Date();
@@ -70,26 +56,6 @@ class StorageS3 extends StorageBase {
             // Anonymize only client IP
             ip = `${anonymizeIp(ip.substring(0, commaIndex))}${ip.substring(commaIndex, ip.length)}`;
         }
-        const sources = this.getSources(JSON.parse(item.config).content);
-        const metadata = {
-            M: {
-                editor_count: {
-                    N: sources.length.toString()
-                }
-            }
-        };
-        if (sources.length === 1) {
-            metadata.M.single_code = {
-                M: {
-                    source: {
-                        S: sources[0].source
-                    },
-                    language: {
-                        S: sources[0].language
-                    }
-                }
-            };
-        }
         now.setSeconds(0, 0);
         return Promise.all([
             this.dynamoDb.putItem({
@@ -109,7 +75,6 @@ class StorageS3 extends StorageBase {
                             clicks: {N: '0'}
                         }
                     },
-                    metadata: metadata,
                     creation_ip: {
                         S: ip
                     },
@@ -192,15 +157,13 @@ class StorageS3 extends StorageBase {
             .then(item => {
                 const full_hash = item.Attributes.full_hash.S;
                 const special_metadata = item.Attributes.named_metadata ? item.Attributes.named_metadata.M : null;
-                const metadata = item.Attributes.metadata ? item.Attributes.metadata.M : null;
                 return this.s3.get(full_hash, this.prefix)
                     .then(result => {
                         // If we're here, we are pretty confident there is a match. But never hurts to double check
                         if (result.hit) {
                             return {
                                 config: result.data.toString(),
-                                specialMetadata: special_metadata,
-                                metadata: metadata
+                                specialMetadata: special_metadata
                             };
                         } else {
                             throw new Error(`ID ${id} not present in storage`);

lib/utils.js

@@ -123,3 +123,23 @@ exports.getHash = function getHash(object, HashVersion = 'Compiler Explorer Defa
         .update(asString)
         .digest('hex');
 };
+
+/***
+ * Gets every pair of source, lang available
+ * @param content {Array}
+ * @returns {Array}
+ */
+exports.glGetEditorSources = function glGetEditorSources(content) {
+    let sources = [];
+    _.each(content, element => {
+        if (element.type === 'component') {
+            if (element.componentName === 'codeEditor') {
+                sources.push({source: element.componentState.source, language: element.componentState.lang});
+            }
+        } else {
+            sources = glGetEditorSources(element.content).concat(sources);
+        }
+    });
+    return sources;
+
+};

test/storage/storage-s3-tests.js

@@ -162,14 +162,14 @@ describe('Stores to s3', () => {
             prefix: "ABCDEF",
             uniqueSubHash: "ABCDEFG",
             fullHash: "ABCDEFGHIJKLMNOP",
-            config: '{"contents": []}'
+            config: 'yo'
         };
 
         const ran = {s3: false, dynamo: false};
         s3PutObjectHandlers.push((q) => {
             q.Bucket.should.equal('bucket');
             q.Key.should.equal('prefix/ABCDEFGHIJKLMNOP');
-            q.Body.should.equal('{"contents": []}');
+            q.Body.should.equal('yo');
             ran.s3 = true;
             return {};
         });
@@ -182,8 +182,7 @@ describe('Stores to s3', () => {
                 stats: {M: {clicks: {N: '0'}}},
                 creation_ip: {S: 'localhost'},
                 // Cheat the date
-                creation_date: {S: q.Item.creation_date.S},
-                metadata: {M: {editor_count: {N: "0"}}}
+                creation_date: {S: q.Item.creation_date.S}
             });
             ran.dynamo = true;
             return {};