added defaultCollections to utils

Author: rotem-codefresh

2021-08-17-2021-08-17-logs.json

@@ -0,0 +1 @@
+[{"_id":"611bb38611ad1e8e4c351797","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Running Codefresh YAML validation\r\n","time":1629205382647},{"_id":"611bb38611ad1e8e4c351798","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"YAML validation passed successfully\r\n","time":1629205382738},{"_id":"611bb3981c7b4c0011783aa4","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Validating connection to Docker daemon...\r\n","time":1629205400975},{"_id":"611bb3991c7b4c0011783aa5","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Connection to Docker daemon validated\r\n","time":1629205401050},{"_id":"611bb3991c7b4c0011783aa6","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Creating logging service...\r\n","time":1629205401091},{"_id":"611bb3991c7b4c0011783aa7","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Logging service created\r\n","time":1629205401494},{"_id":"611bb3991c7b4c0011783aa8","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Validating logging service\r\n","time":1629205401500},{"_id":"611bb39a1c7b4c0011783aa9","slot":"metrics.memory","payload":{"time":1629205402720,"usage":60108800},"accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791"},{"_id":"611bb39b1c7b4c0011783aaa","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Logging service validated\r\n","time":1629205403964},{"_id":"611bb39b1c7b4c0011783aab","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Provisioning volume: workflow_611bb38511ad1e8e4c351792\r\n","time":1629205403995},{"_id":"611bb39c1c7b4c0011783aac","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Successfully provisioned volume: workflow_611bb38511ad1e8e4c351792\r\n","time":1629205404011},{"_id":"611bb39c1c7b4c0011783aad","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Creating environment variable exporting file: /codefresh/volume/env_vars_to_export\r\n","time":1629205404026},{"_id":"611bb39c1c7b4c0011783aae","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Creating export environment variable script file: /codefresh/volume/cf_export\r\n","time":1629205404539},{"_id":"611bb39c1c7b4c0011783aaf","slot":"metrics.memory","payload":{"time":1629205404728,"usage":76705792},"accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791"},{"_id":"611bb39d1c7b4c0011783ab0","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Creating debug environment script file: /codefresh/volume/setup_debugger.sh\r\n","time":1629205405052},{"_id":"611bb39d1c7b4c0011783ab1","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Requesting account clusters\r\n","time":1629205405624},{"_id":"611bb39d1c7b4c0011783ab2","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.Initializing Process.logs","payload":"Scheduling health checks\r\n","time":1629205405768},{"_id":"611bb39e1c7b4c0011783ab3","slot":"metrics.memory","payload":{"time":1629205406736,"usage":77905920},"accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791"},{"_id":"611bb3a01c7b4c0011783ab4","slot":"metrics.memory","payload":{"time":1629205408742,"usage":79290368},"accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791"},{"_id":"611bb3a21c7b4c0011783ab5","slot":"metrics.memory","payload":{"time":1629205410758,"usage":99737600},"accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791"},{"_id":"611bb3a285480800174e2711","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.test.logs","payload":"Pulling image library/ubuntu:latest\r\n","time":1629205410292},{"_id":"611bb3a285480800174e2712","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.test.logs","payload":"Digest: sha256:82becede498899ec668628e7cb0ad87b6e1c371cb8a1e597d83a47fac21d6af3\r\n","time":1629205410947},{"_id":"611bb3a285480800174e2713","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.test.logs","payload":"Status: Image is up to date for ubuntu:latest\r\n","time":1629205410954},{"_id":"611bb3a31c7b4c0011783ab6","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.test.logs","payload":"Successfully ran freestyle step: Running test\r\n","time":1629205411731},{"_id":"611bb3a31c7b4c0011783ab7","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.test.logs","payload":"Reading environment variable exporting file contents.\r\n","time":1629205411738},{"_id":"611bb3a31c7b4c0011783ab8","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.test.logs","payload":"Reading environment variable exporting file contents.\r\n","time":1629205411892},{"_id":"611bb3a385480800174e2714","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.test.logs","payload":"------------------------------\r\n\u001b[32mExecuting command:\u001b[39m echo hi\r\n","time":1629205411580},{"_id":"611bb3a385480800174e2715","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","slot":"steps.test.logs","payload":"hi\r\n","time":1629205411595},{"_id":"611bb3a41c7b4c0011783ab9","slot":"metrics.memory","payload":{"time":1629205412778,"usage":81248256},"accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791"}]

2021-08-17-2021-08-17-metadata.json

@@ -0,0 +1 @@
+[{"_id":"611bb3855651001a26d35c0b","accountId":"6106c9362e4b3e7f21f0b113","jobId":"611bb38511ad1e8e4c351791","id":"611bb38511ad1e8e4c351791","visibility":"private","steps":{"Initializing Process":{"name":"Initializing Process","status":"success","finishTimeStamp":1629205406,"creationTimeStamp":1629205401},"test":{"name":"test","status":"success","creationTimeStamp":1629205406,"finishTimeStamp":1629205412,"metrics":{"logs":{"total":74}}}},"lastUpdate":1629205412492,"data":{"keepPVCsForPendingApproval":false,"created":1629205401,"started":1629205401,"finished":1629205413},"status":"success","metrics":{"logs":{"total":74}}}]

lib/interface/cli/commands/offline-logs/ensure-index.cmd.js

@@ -1,7 +1,7 @@
 const { MongoClient } = require("mongodb");
 const Command = require("../../Command");
 const cmd = require("./base.cmd");
-const { ensureIndex } = require('./utils')
+const { ensureIndex, defaultCollections } = require('./utils')
 
 const command = new Command({
   command: "ensure-index",
@@ -18,12 +18,11 @@ const command = new Command({
     ),
   handler: async (argv) => {
     const { uri, db } = argv;
-    const collections = ["logs", "metadata"];
     const client = new MongoClient(uri);
     await client.connect();
     const database = client.db(db);
     const failedCollections = [];
-    for (const collection of collections) {
+    for (const collection of defaultCollections) {
       try {
         await ensureIndex(database, collection);
       } catch (error) {

lib/interface/cli/commands/offline-logs/ensure-index.spec.js

@@ -1,12 +1,16 @@
-const readline = require('readline')
+const {Db, Collection} = require('mongodb');
+const { Collection } = require('yaml/types');
 const { ensureIndex } = require('./utils')
 
 describe("ensure-index", () => {
   describe("ensureIndex", () => {
     it("gets user input from prompt and returns true/false", () => {
-      
-      const collection = 
-      ensureIndex(database, collection)
+      Db.collection() = jest.fn();
+      Collection.createIndexes() = jest.fn()
+
+      const database = new Db('db');
+      const collection = 'whatever';
+      ensureIndex(database, collection);
       });
     });
   });

lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js

@@ -2,7 +2,7 @@ const { MongoClient, ObjectId } = require("mongodb");
 const moment = require('moment')
 const Command = require('../../Command');
 const cmd = require('./base.cmd');
-const { objectIdFromDate } = require('./utils')
+const { objectIdFromDate, defaultCollections } = require('./utils')
 
 const offloadToCollection = async function(sourceDBObj, collection, targetDB, cutoffDate) {
   const sourceCollectionObj = sourceDBObj.collection(collection)
@@ -70,13 +70,12 @@ const command = new Command({
         cutoffDate,
       } = argv
 
-      const collections = ["logs", "metadata"];
       const client = new MongoClient(uri);
       try{
         await client.connect()
         const failedCollections = [];
         const sourceDBObj = client.db(db);
-        const promises = collections.map( async (collection) => {
+        const promises = defaultCollections.map( async (collection) => {
           try{
             await offloadToCollection(sourceDBObj, collection, targetDB || db, cutoffDate);
           } catch (error) {

lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js

@@ -3,8 +3,7 @@ const { MongoClient, ObjectId } = require("mongodb");
 const moment = require('moment')
 const Command = require('../../Command');
 const cmd = require('./base.cmd');
-const {objectIdFromDate, writeLogsToFile, checkRemnant} = require('./utils')
-const { boolean } = require("yargs");
+const {objectIdFromDate, writeLogsToFile, checkRemnant, defaultCollections} = require('./utils');
 
 const offloadToFile = async function(database, collection, chunkDuration, cutoffDate, path) {
   const collectionObj = database.collection(collection);
@@ -108,7 +107,7 @@ const command = new Command({
       .option('collections', {
         alias: 'c',
         describe: "Source collections names",
-        default: ["logs", "metadata"],
+        default: defaultCollections,
         array: true,
         type: "string",
         coerce(arg) {

lib/interface/cli/commands/offline-logs/utils.js

@@ -4,6 +4,8 @@ const moment = require("moment");
 const { join } = require("path");
 const fs = require("fs");
 
+const defaultCollections = ["logs", "metadata"];
+
 // converts date to objectId for filter purposes
 const objectIdFromDate = function (date) {
   return Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
@@ -24,6 +26,7 @@ const writeLogsToFile = function (
   const absPath = join(path, fileName);
   const data = JSON.stringify(logsToArchive);
   fs.writeFileSync(absPath, data);
+  console.info(`logs from collection '${collection}' were offloaded to '${absPath}'`)
 };
 
 const checkRemnant = function (lowerBound, cutoffDateObj) {
@@ -103,4 +106,5 @@ module.exports = {
   checkRemnant,
   ensureIndex,
   getUserInput,
+  defaultCollections,
 };