Upload json tagging data, update upload/download functions to use singular data_access library and configuration. (#35)

* Upload flow reworked to use new db_accessv2. Fixing db update bug from string subsitution. Upload handles visit&tag, visit&notag, novisit

Saving untagged to INCOMPLETE_TAG state works through local func testing

cleaned up saving untagged images a bit

Saving tags to DB also works

Working upload flow with visit&tag, visit&notag, novisit

removing creds

restructing vott_json_parser file. Deleting jsonpickle version

Moving things around for importclean up purposes. Upload function still working, need to refactor download and check prior existing tests

pytest working, had to rename db_access_v1 to fix conflicts

Some more cleanup and restructring. Tested working through upload function and pytest

* Working on refactoring download function, saving work. Need to get images from db then return

Reverting some accidental renaming in other directories

Removing vscode files

download working, still need to 1. combine db config for all 3 functions, 2. write tests

Download now works with new db_access_v2. Need to create shared_db config

ditto prior

* single db configuration

minor spacing

* updating onboarding function to use new shared directory

* temp commenting out onboarding/onboarding-client.py as it breaks pytests. Should probably remove hardcoded resources

* feedback

* minor TODO comment reshuffling

* removing deprecated db_access_v1

Author: mtarng

.gitignore

@@ -118,3 +118,6 @@ tag/*.csv
 
 # TF exported graph files
 .pb
+
+# VSCode
+.vscode/*.json

functions/pipeline/download/__init__.py

@@ -2,39 +2,54 @@
 
 import azure.functions as func
 import json
-import os
 
-from ..shared import db_access as DB_Access
-from ..shared import vott_json_parser as vott_json_parser
+from ..shared.vott_parser import create_starting_vott_json
+from ..shared.db_provider import get_postgres_provider
+from ..shared.db_access import ImageTagDataAccess
+
 
 def main(req: func.HttpRequest) -> func.HttpResponse:
     logging.info('Python HTTP trigger function processed a request.')
 
-    imageCount = req.params.get('imageCount')
+    image_count = int(req.params.get('imageCount'))
+    user_id = int(req.params.get('userId'))
+
     # setup response object
     headers = {
         "content-type": "application/json"
     }
-    if not imageCount:
+    if not user_id:
+        return func.HttpResponse(
+            status_code=401,
+            headers=headers,
+            body=json.dumps({"error": "invalid userId given or omitted"})
+        )
+    elif not image_count:
         return func.HttpResponse(
             status_code=400,
             headers=headers,
             body=json.dumps({"error": "image count not specified"})
         )
     else:
-        # setup response object
-        connection = DB_Access.get_connection()
-        # TODO: images need more meaningful data than just download urls
-        image_urls = DB_Access.get_images_for_tagging(connection, imageCount)
+        try:
+            # DB configuration
+            data_access = ImageTagDataAccess(get_postgres_provider())
 
-        # TODO: Build vott json
-        vott_json = vott_json_parser.create_starting_json(image_urls)
+            image_urls = list(data_access.get_new_images(image_count, user_id))
 
-        return_body_json = {"imageUrls": image_urls, "vottJson": vott_json}
+            # TODO: Populate starting json with tags, if any exist... (precomputed or retagging?)
+            vott_json = create_starting_vott_json(image_urls)
 
-        content = json.dumps(return_body_json)
-        return func.HttpResponse(
-            status_code=200, 
-            headers=headers, 
-            body=content
-        )
+            return_body_json = {"imageUrls": image_urls, "vottJson": vott_json}
+
+            content = json.dumps(return_body_json)
+            return func.HttpResponse(
+                status_code=200,
+                headers=headers,
+                body=content
+            )
+        except Exception as e:
+            return func.HttpResponse(
+                "exception:" + str(e),
+                status_code=500
+            )

functions/pipeline/onboarding/__init__.py

@@ -1,18 +1,15 @@
 import os
 import logging
 import azure.functions as func
-from ..shared import db_access_v2 as DB_Access_V2
-from azure.storage.blob import BlockBlobService, ContentSettings
+
+from ..shared.db_provider import get_postgres_provider
+from ..shared.db_access import ImageTagDataAccess, ImageInfo
+from azure.storage.blob import BlockBlobService
 
 # TODO: User id as param to function - holding off until further discussion
 # regarding whether user ID should be generated/looked up by the CLI or
 # from within this function
 
-default_db_host = ""
-default_db_name = ""
-default_db_user = ""
-default_db_pass = ""
-
 def main(req: func.HttpRequest) -> func.HttpResponse:
     logging.info('Python HTTP trigger function processed a request.')
 
@@ -41,19 +38,17 @@ def main(req: func.HttpRequest) -> func.HttpResponse:
         # Create ImageInfo object (def in db_access.py)
         # Note: For testing, default image height/width are set to 50x50
         # TODO: Figure out where actual height/width need to come from
-        image = DB_Access_V2.ImageInfo(original_filename, url, 50, 50)
+        image = ImageInfo(original_filename, url, 50, 50)
         # Append image object to the list
         image_object_list.append(image)
 
     # TODO: Wrap db access section in try/catch, send an appropriate http response in the event of an error
     logging.info("Now connecting to database...")
-    db_config = DB_Access_V2.DatabaseInfo(os.getenv('DB_HOST', default_db_host), os.getenv('DB_NAME', default_db_name), os.getenv('DB_USER', default_db_user), os.getenv('DB_PASS', default_db_pass))
-    data_access = DB_Access_V2.ImageTagDataAccess(DB_Access_V2.PostGresProvider(db_config))
+    data_access = ImageTagDataAccess(get_postgres_provider())
     logging.info("Connected.")
 
     # Create user id
-    user_id = data_access.create_user(DB_Access_V2.getpass.getuser())
-    logging.info("The user id for '{0}' is {1}".format(DB_Access_V2.getpass.getuser(),user_id))
+    user_id = data_access.create_user("testuser")  # TODO: remove this hardcoding, should be passed in the request.
 
     # Add new images to the database, and retrieve a dictionary ImageId's mapped to ImageUrl's
     image_id_url_map = data_access.add_new_images(image_object_list,user_id)

functions/pipeline/onboarding/onboarding-client.py

@@ -1,41 +1,39 @@
-import requests
-import json
-import pg8000
-
-# The following mock client imitates the CLI during the onboarding scenario for new images.
-# The expectation is that the CLI uploads images to a temporary blob store, then gets a list 
-# of URLs to those images and passes the list to an HTTP trigger function in the format of
-# a JSON string.  The HTTP trigger function creates rows in the database for the images,
-# retrieves the ImageId's for them, and then copies the images, each renamed as "ImageId.extension",
-# into a permanent blob storage container.  The HTTP function returns the list of URLs to
-# the images in permanent blob storage.
-
-print("\nTest client for CLI Onboarding scenario")
-print('-' * 40)
-
-# functionURL = "https://onboardinghttptrigger.azurewebsites.net/api/onboarding?code=lI1zl4IhiHcOcxTS85RsE7yZJXeNRxnr7tXSO1SrLWdpiN0W6hT3Jw=="
-functionURL = "http://localhost:7071/api/onboarding"
-# Sean's function URL:
-# functionURL = "https://onboardinghttptrigger.azurewebsites.net/api/onboarding?code=lI1zl4IhiHcOcxTS85RsE7yZJXeNRxnr7tXSO1SrLWdpiN0W6hT3Jw=="
-# functionURL = "https://abrig-linux-func.azurewebsites.net/api/onboarding"
-
-urlList = { "imageUrls": ["https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies1.jpg",
-                         "https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies2.jpg",
-                         "https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies3.jpg"] }
-
-headers = {"Content-Type": "application/json"}
-
-print("Now executing POST request to onboard images...to:")
-print("Function URL: " + functionURL)
-print("Headers:")
-for key, value in headers.items():
-    print("\t" + key + ": " + value)
-response = requests.post(url=functionURL, headers=headers, json=urlList)
-print("Completed POST request.")
-
-raw_response = response.text
-response_array = raw_response.split(", ")
-response_output = "\n".join(response_array)
-
-print(f"Response status code: {response.status_code}")
-print(f"Response string: {response_output}")
+# import requests
+#
+# # The following mock client imitates the CLI during the onboarding scenario for new images.
+# # The expectation is that the CLI uploads images to a temporary blob store, then gets a list
+# # of URLs to those images and passes the list to an HTTP trigger function in the format of
+# # a JSON string.  The HTTP trigger function creates rows in the database for the images,
+# # retrieves the ImageId's for them, and then copies the images, each renamed as "ImageId.extension",
+# # into a permanent blob storage container.  The HTTP function returns the list of URLs to
+# # the images in permanent blob storage.
+#
+# print("\nTest client for CLI Onboarding scenario")
+# print('-' * 40)
+#
+# # functionURL = "https://onboardinghttptrigger.azurewebsites.net/api/onboarding?code=lI1zl4IhiHcOcxTS85RsE7yZJXeNRxnr7tXSO1SrLWdpiN0W6hT3Jw=="
+# functionURL = "http://localhost:7071/api/onboarding"
+# # Sean's function URL:
+# # functionURL = "https://onboardinghttptrigger.azurewebsites.net/api/onboarding?code=lI1zl4IhiHcOcxTS85RsE7yZJXeNRxnr7tXSO1SrLWdpiN0W6hT3Jw=="
+# # functionURL = "https://abrig-linux-func.azurewebsites.net/api/onboarding"
+#
+# urlList = { "imageUrls": ["https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies1.jpg",
+#                          "https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies2.jpg",
+#                          "https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies3.jpg"] }
+#
+# headers = {"Content-Type": "application/json"}
+#
+# print("Now executing POST request to onboard images...to:")
+# print("Function URL: " + functionURL)
+# print("Headers:")
+# for key, value in headers.items():
+#     print("\t" + key + ": " + value)
+# response = requests.post(url=functionURL, headers=headers, json=urlList)
+# print("Completed POST request.")
+#
+# raw_response = response.text
+# response_array = raw_response.split(", ")
+# response_output = "\n".join(response_array)
+#
+# print(f"Response status code: {response.status_code}")
+# print(f"Response string: {response_output}")

functions/pipeline/shared/__init__.py

@@ -0,0 +1 @@
+from .db_access_v2 import ImageTagDataAccess, ImageTag, ImageInfo, ImageTagState

functions/pipeline/shared/db_access.py

@@ -6,7 +6,8 @@
 from enum import IntEnum, unique
 import getpass
 import itertools
-from .db_provider import DatabaseInfo, PostGresProvider
+
+from ..db_provider import DatabaseInfo, PostGresProvider
 
 @unique
 class ImageTagState(IntEnum):
@@ -76,11 +77,10 @@ def get_new_images(self, number_of_images, user_id):
             conn = self._db_provider.get_connection()
             try:
                 cursor = conn.cursor()
-                # TODO: Should we add TagStateId = INCOMPLETE_TAG also for fetching images?
                 query = ("SELECT b.ImageId, b.ImageLocation, a.TagStateId FROM Image_Tagging_State a "
-                        "JOIN Image_Info b ON a.ImageId = b.ImageId WHERE a.TagStateId = 1 order by "
+                        "JOIN Image_Info b ON a.ImageId = b.ImageId WHERE a.TagStateId IN ({1}, {2}) order by "
                         "a.createddtim DESC limit {0}")
-                cursor.execute(query.format(number_of_images))
+                cursor.execute(query.format(number_of_images, ImageTagState.READY_TO_TAG, ImageTagState.INCOMPLETE_TAG))
                 for row in cursor:
                     print('Image Id: {0} \t\tImage Name: {1} \t\tTag State: {2}'.format(row[0], row[1], row[2]))
                     selected_images_to_tag[str(row[0])] = str(row[1])

functions/pipeline/shared/db_access/__init__.py

@@ -2,7 +2,7 @@
 from unittest.mock import patch 
 from unittest.mock import Mock
 
-from db_access_v2 import(
+from .db_access_v2 import(
     ImageTagDataAccess,
     ArgumentException,
     ImageTagState,

functions/pipeline/shared/db_access_v2.py …ipeline/shared/db_access/db_access_v2.pyfunctions/pipeline/shared/db_access_v2.py renamed to functions/pipeline/shared/db_access/db_access_v2.py functions/pipeline/shared/db_access_v2.py renamed to functions/pipeline/shared/db_access/db_access_v2.py

@@ -0,0 +1 @@
+from .db_provider import DatabaseInfo, DBProvider, PostGresProvider, get_postgres_provider