GPII-3138: Update snapsets in data base

scripts/convertPrefs.js

@@ -9,7 +9,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt
 */
 
 // This script reads files from an input directory that contains preferences JSON5 files and convert them to JSON files of GPII keys and
-// preferences safes suitable for direct loading into CouchDB or PouchDB, which comply with the new GPII data model: 
+// preferences safes suitable for direct loading into CouchDB or PouchDB, which comply with the new GPII data model:
 // https://wiki.gpii.net/w/Keys,_KeyTokens,_and_Preferences in the target directory
 // Usage: node scripts/convertPrefs.js {input_path} {target_path}
 //
@@ -61,7 +61,7 @@ rimraf(targetDir, function () {
                     "_id": prefsSafeId,
                     "type": "prefsSafe",
                     "schemaVersion": "0.1",
-                    "prefsSafeType": "user",
+                    "prefsSafeType": "snapset",
                     "name": gpiiKey,
                     "password": null,
                     "email": null,

scripts/deleteSnapsets.js

@@ -0,0 +1,162 @@
+/*!
+Copyright 2018 OCAD University
+
+Licensed under the New BSD license. You may not use this file except in
+compliance with this License.
+
+You may obtain a copy of the License at
+https://github.com/GPII/universal/blob/master/LICENSE.txt
+*/
+
+// This script modifies the preferences data base:
+// 1. Finds all the Prefs Safes of type "snapset" (prefsSafesType = "snapset"),
+// 2. Finds all the GPII Keys associated with each snapset Prefs Safe
+// 3. Deletes the found Prefs Safes and associated GPII Keys
+//
+// A sample command that runs this script:
+// node deleteSnapsets.js $COUCHDBURL
+
+"use strict";
+
+var http = require("http"),
+    url = require("url"),
+    fluid = require("infusion");
+
+var gpii = fluid.registerNamespace("gpii");
+fluid.registerNamespace("gpii.dataLoader");
+fluid.setLogging(fluid.logLevel.INFO)
+
+var dbLoader = gpii.dataLoader;
+dbLoader.couchDbUrl = process.argv[2];
+if (!fluid.isValue(dbLoader.couchDbUrl)) {
+    fluid.log ("COUCHDB_URL environment variable must be defined");
+    process.exit(1);
+}
+fluid.log("COUCHDB_URL: '" + dbLoader.couchDbUrl + "'");
+dbLoader.prefsSafesViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
+dbLoader.gpiiKeyViewUrl = dbLoader.couchDbUrl + "/_design/views/_view/findGpiiKeysByPrefsSafeId";
+dbLoader.parsedCouchDbUrl = url.parse(dbLoader.couchDbUrl);
+dbLoader.docsToRemove=[];
+
+/**
+ * Find the Prefs Safes of type "snapSet", add them to an array of records to
+ * remove, then use them to find their associated GPII Key(s).
+ * @param {Object} response - The response from the data base containing the
+ *                            snapSet Prefs Safes as a JSON string.
+ */
+dbLoader.processSnapsets = function (response) {
+    var snapSetsString = "";
+    response.setEncoding("utf8");
+    response.on("data", function (chunk) {
+        snapSetsString += chunk;
+    });
+    response.on("end", function () {
+        dbLoader.snapSets = JSON.parse(snapSetsString);
+        fluid.each(dbLoader.snapSets.rows, function (aSnapset) {
+            aSnapset.value._deleted = true;
+            dbLoader.docsToRemove.push(aSnapset.value);
+        });
+        dbLoader.addGpiiKeysAndBulkDelete(dbLoader.snapSets.rows, dbLoader.docsToRemove);
+    });
+};
+
+/**
+ * Find the GPII key records associated with the given snapset Prefs Safes,
+ * delete each one as found, and then batch delete all the snapset Prefs Safes.
+ * @param {Array} snapSets      - The snapsets of interest.
+ * @param {Array} docsToRemove  - Array of records to delete.
+ */
+dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) {
+    fluid.each(snapSets, function (aSnapset) {
+        var gpiiKeyId = aSnapset.value._id;
+        fluid.log("Snapset: " + gpiiKeyId);
+        var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22";
+        var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) {
+            var respString = "";
+            resp.setEncoding("utf8");
+            resp.on("data", function (chunk) {
+                respString += chunk;
+            });
+            // This response "end" event doesn't finish until after the call to
+            // doBatchDelete() below, which is too late to add the GPII Key to
+            // docsToRemove and have them removed.
+            // Note - if there is a way to determine the final "end" event, then
+            // it could  call doBatchDelete().
+            resp.on("end", function () {
+                var gpiiKeyRecords = JSON.parse(respString);
+                fluid.each(gpiiKeyRecords.rows, function (gpiiKey) {
+                    fluid.log("GPII Key: " + gpiiKey.value._id);
+                    dbLoader.deleteGpiiKey(gpiiKey.value);
+                });
+            });
+        });
+        getGpiiKeysRequest.on("error", function (e) {
+            fluid.log("Error finding snapsets' associated GPII Keys: " + e.message);
+        });
+        getGpiiKeysRequest.end();
+    });
+    dbLoader.doBatchDelete(docsToRemove);
+};
+
+/**
+ * Delete the given GPII Key record.
+ * @param {Object} gpiiKey  - The key to delete.
+ */
+dbLoader.deleteGpiiKey = function (gpiiKey) {
+    var deleteOptions = {
+        hostname: dbLoader.parsedCouchDbUrl.hostname,
+        port: dbLoader.parsedCouchDbUrl.port,
+        path: "",           // filled in below.
+        method: "DELETE",
+        headers: {
+            "Accept": "application/json"
+        }
+    };
+    deleteOptions.path = "/gpii/" + gpiiKey._id + "?rev=" + gpiiKey._rev;
+    var deleteGpiiKeyRequest = http.request(deleteOptions, function (res) {
+        fluid.log("STATUS: " + res.statusCode);
+        fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2));
+    });
+    deleteGpiiKeyRequest.on("error", function (e) {
+        fluid.log("Error finding GPII Key: " + e.message);
+    });
+    deleteGpiiKeyRequest.end();
+};
+
+/**
+ * Delete the snapset Prefs Safes and their associated GPII Keys.
+ * @param {Array} docsToRemove  - Array of records to delete.
+ */
+dbLoader.doBatchDelete = function (docsToRemove) {
+    var batchDeleteOptions = {
+        hostname: dbLoader.parsedCouchDbUrl.hostname,
+        port: dbLoader.parsedCouchDbUrl.port,
+        path: "/gpii/_bulk_docs",
+        method: "POST",
+        headers: {
+            "Accept": "application/json",
+            "Content-Length": 0,          // filled in below
+            "Content-Type": "application/json"
+        }
+    };
+    var batchPostData = JSON.stringify({"docs": docsToRemove});
+    batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData);
+    var batchDeleteReq = http.request(batchDeleteOptions, function (res) {
+        fluid.log("STATUS: " + res.statusCode);
+        fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2));
+        res.on('end', function () {
+            fluid.log('Batch deletion of snapsets');
+        });
+    });
+    batchDeleteReq.on('error', function (e) {
+        fluid.log("Error deleting snapset Prefs Safes: " + e.message);
+    });
+    batchDeleteReq.write(batchPostData);
+    batchDeleteReq.end();
+};
+
+dbLoader.snapSetsRequest = http.request(dbLoader.prefsSafesViewUrl, dbLoader.processSnapsets);
+dbLoader.snapSetsRequest.on("error", function (e) {
+    fluid.log("Error finding snapsets Prefs Safes: " + e.message);
+});
+dbLoader.snapSetsRequest.end();

scripts/vagrantCloudBasedContainers.sh

@@ -32,9 +32,15 @@ COUCHDB_IMAGE=couchdb
 COUCHDB_PORT=5984
 COUCHDB_HEALTHCHECK_DELAY=2
 COUCHDB_HEALTHCHECK_TIMEOUT=30
+if [ "$NO_REBUILD" == "true" ] ; then
+    CLEAR_INDEX=
+else
+    CLEAR_INDEX=1
+fi
 
-STATIC_DATA_DIR="/home/vagrant/sync/universal/testData/dbData"
-BUILD_DATA_DIR="/home/vagrant/sync/universal/build/dbData"
+UNIVERSAL_DIR="/home/vagrant/sync/universal"
+STATIC_DATA_DIR="$UNIVERSAL_DIR/testData/dbData"
+BUILD_DATA_DIR="$UNIVERSAL_DIR/build/dbData"
 
 DATALOADER_IMAGE="gpii/gpii-dataloader"
 DATALOADER_COUCHDB_URL="http://couchdb:${COUCHDB_PORT}/gpii"
@@ -77,7 +83,7 @@ docker run -d -p $COUCHDB_PORT:$COUCHDB_PORT --name couchdb $COUCHDB_IMAGE
 wget -O /dev/null --retry-connrefused --waitretry=$COUCHDB_HEALTHCHECK_DELAY --read-timeout=20 --timeout=1 --tries=$COUCHDB_HEALTHCHECK_TIMEOUT http://localhost:$COUCHDB_PORT
 
 # Load the CouchDB data
-docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=1 $DATALOADER_IMAGE
+docker run --rm --link couchdb -v $STATIC_DATA_DIR:/static_data -e STATIC_DATA_DIR=/static_data -v $BUILD_DATA_DIR:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=$DATALOADER_COUCHDB_URL -e CLEAR_INDEX=$CLEAR_INDEX -v $UNIVERSAL_DIR:/universal -e NODE_PATH=/universal $DATALOADER_IMAGE
 
 # Wait for the CouchDB views become accessible. Accessing the view URL forced the view index to build which take time.
 # The URL returns 500 when the index is not ready, so use "--retry-on-http-error" option to continue retries at 500 response code.

testData/dbData/views.json

@@ -10,6 +10,12 @@
             },
             "findAuthorizationByAccessToken": {
                 "map": "function(doc) {if (doc.type === 'gpiiAppInstallationAuthorization' && doc.revoked === false) {emit(doc.accessToken, {'_id': doc.clientId, 'authorization': doc})}}"
+            },
+            "findSnapsetPrefsSafes": {
+                "map": "function(doc) {if (doc.type === 'prefsSafe' && doc.prefsSafeType === 'snapset') {emit(doc._id, doc)}}"
+            },
+            "findGpiiKeysByPrefsSafeId": {
+                "map": "function(doc) {if (doc.type === 'gpiiKey') {emit(doc.prefsSafeId, doc)}}"
             }
         }
     }