initial commit for effective docker proposal

more_effective_docker/Dockerfile.use-installer-user

@@ -0,0 +1,59 @@
+FROM ubuntu:20.04 as builder
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/Los_Angeles
+
+RUN apt-get update && apt-get install -y \
+    libcanberra-gtk-module \
+    wget \
+    xz-utils && \
+    rm -rf /var/lib/apt/lists/*
+
+# make /bin/sh symlink to bash instead of dash:
+RUN echo "dash dash/sh boolean false" | debconf-set-selections
+RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash
+
+# add ARGs so that these can be passed in while doing a docker build, the following will be default values
+ARG TAG=v0.0.0
+ARG USER=docker
+ARG UID=1000
+ARG GID=1000
+# default password for user
+ARG PW=docker
+
+# using unencrypted password/specifying password
+RUN useradd -m ${USER} --uid=${UID} && echo "${USER}:${PW}" | \
+      chpasswd
+
+# using the same encrypted password as host (doesn't work yet)
+#COPY /etc/group /etc/group 
+#COPY /etc/passwd /etc/passwd
+#COPY /etc/shadow /etc/shadow
+
+# [unused] : run this as root user before changing to local user - removing fixed entrypoint for now
+#COPY ./docker-entrypoint.sh /
+#RUN chmod +x /docker-entrypoint.sh
+
+
+ENV INSTALL_DIR=/opt/symbiflow/eos-s3/$TAG
+
+#  create dir and chown to local user
+RUN mkdir /opt/symbiflow
+RUN chown ${USER} /opt/symbiflow
+
+# become local user for installation
+USER ${UID}:${GID}
+WORKDIR /home/${USER}
+
+# continue with installation
+RUN wget https://github.com/QuickLogic-Corp/quicklogic-fpga-toolchain/releases/download/$TAG/Symbiflow_$TAG.gz.run
+RUN chmod 755 Symbiflow_$TAG.gz.run
+RUN ./Symbiflow_$TAG.gz.run
+RUN rm Symbiflow_$TAG.gz.run
+
+ENV PATH="${INSTALL_DIR}/install/bin:${INSTALL_DIR}/install/bin/python:$PATH"
+
+# [unused] we can always specify the entrypoint at runtime, so default would be to drop into a bash session.
+#ENTRYPOINT ["/docker-entrypoint.sh"]
+
+

more_effective_docker/README.md

@@ -0,0 +1,166 @@
+# Using the FPGA Toolchain effectively with Docker
+
+Aiming at end users of the QF board for development, using M4 + FPGA, it would probably be easier to use the FPGA Toolchain, if we combine it with Docker.
+It would be easier to switch between versions of the toolchain, and is equivalent to a new installation with the installer as well, while providing a way to prevent dependencies of the toolchain to become a problem to maintain, as in different versions of the toolchain may have 
+
+## Dockerfile
+
+The current Dockerfile (Dockerfile.use-installer) - has one problem, in that the installer is being run with root user, and hence we cannot use bind mounts to the host system to effectively use it. With bind mounts, we can have the qorc-sdk or gateware in the host system, and use the toolchain to build it, and the outputs are available in the host system just as if we used a locally installed FPGA toolchain.
+With the current Dockerfile based image, if we use bind mounts, then the files/dirs created will be having root as owner and access permissions locked as well, and current user in the host system cannot use it, so it is ok for testing the installation, but not for usage by the end user or developer.
+
+Hence, it would be good to modify it so that the current user UID, GID are passed in to the build, and the actual installation of the toolchain happens under the current user, effectively making it equivalent to a local installation.
+```
+...
+# make /bin/sh symlink to bash instead of dash:
+RUN echo "dash dash/sh boolean false" | debconf-set-selections
+RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash
+
+# MOD 1. add ARGs so that these can be passed in while doing a docker build, the following will be default values
+# TAG should refer to the Symbiflow Toochain Version tag: for e.g. v1.2.0
+ARG TAG=v1.2.0
+ARG USER=docker
+ARG UID=1000
+ARG GID=1000
+# default password for user
+ARG PW=docker
+
+RUN useradd -m ${USER} --uid=${UID} && echo "${USER}:${PW}" | \
+      chpasswd
+
+ENV INSTALL_DIR=/opt/symbiflow/eos-s3/$TAG
+
+# MOD 2: create dir and chown to local user
+RUN mkdir /opt/symbiflow
+RUN chown ${USER} /opt/symbiflow
+
+# MOD 3: become local user for installation
+USER ${UID}:${GID}
+WORKDIR /home/${USER}
+
+# continue with installation as before
+RUN wget https://github.com/QuickLogic-Corp/quicklogic-fpga-toolchain/releases/download/$TAG/Symbiflow_$TAG.gz.run
+
+...
+```
+
+For full Dockerfile refer to **Dockerfile.use-installer-user**
+
+## Building the Docker image
+
+As the current user, the image can be built with:
+```
+docker build --build-arg USER=$USER \
+         --build-arg UID=$UID \
+         --build-arg GID=$GID \
+         --build-arg PW=password \
+         --build-arg TAG=v1.2.0 \
+         -t ql_symbiflow_user:v1.2.0 \
+         -f Dockerfile.use-installer-user\
+         .
+```
+
+The docker tag `ql_symbiflow_user:1.2.0` can be modified as per needs.
+Ideally, it would be great if the Symbiflow Toolchain version can be `1.2.0` rather than `v1.2.0` so we can use the same for both the docker image tag, as well as the toolchain version.
+This (example) build script is at: **docker-ql-symbiflow-build-with-current-user.sh**
+For other versions, TAG and the docker -t value will change.
+
+## Using the Docker image
+
+For the most convenient method of using docker images as a "portable toolchain", while having the input sources on the host system, and the build outputs on the host system as well, we have a few things we set to make it painless:
+1. we use bind mounts, to map the host system sources directory to the docker container.
+2. we map the current user to the docker container, and run as the same user
+3. we provide a method to directly pass in commands to the `docker run` so that the container spins up, executes the toolchain process, generates output and exits on its own
+
+For our FPGA toolchain, as an example, assume that we have the FPGA designs in the following location:
+`~/work/quicklogic/fpga`
+we have 2 examples copied here, one from the toolchain tests, and one from qorc-sdk.
+`counter_16bit`
+`fpga_helloworldhw`
+Both of these are in the **example_fpga_designs**
+
+We need an entrypoint for the docker image, which is convenient to be defined as bash script:
+```
+#!/bin/bash
+
+source "$INSTALL_DIR/conda/etc/profile.d/conda.sh"
+conda activate
+
+ql_symbiflow $@
+```
+
+This example entrypoint is at **docker-ql-symbiflow-entrypoint.sh**
+
+Now, using this entrypoint, and adding bind mount + user mapping, we can do:
+```
+docker run -it --rm \
+--volume="/home/$(id -un)/work:/home/$(id -un)/work:rw" \
+--volume="/etc/group:/etc/group:ro" \
+--volume="/etc/passwd:/etc/passwd:ro" \
+--volume="/etc/shadow:/etc/shadow:ro" \
+--user $(id -u):$(id -g) \
+-e USER=$(id -un) \
+-e HOME=/home/$(id -un) \
+--entrypoint=./docker-ql-symbiflow-entrypoint.sh \
+ql_symbiflow_user:v1.2.0 \
+-compile -src work/quicklogic/fpga/fpga_helloworldhw -d ql-eos-s3 -P PU64 -v helloworldfpga.v -t helloworldfpga -p quickfeather.pcf -dump header
+```
+
+We can condense the fixed parts into a sh script like so
+```
+docker run -it --rm \
+--volume="/home/$(id -un)/work:/home/$(id -un)/work:rw" \
+--volume="/etc/group:/etc/group:ro" \
+--volume="/etc/passwd:/etc/passwd:ro" \
+--volume="/etc/shadow:/etc/shadow:ro" \
+--user $(id -u):$(id -g) \
+-e USER=$(id -un) \
+-e HOME=/home/$(id -un) \
+--entrypoint=./docker-ql-symbiflow-entrypoint.sh \
+ql_symbiflow_user:v1.2.0 \
+$@
+```
+This (example script) is at **docker-ql-symbiflow-run-compile.sh**
+
+and use it to compile any FPGA design using :
+```
+docker-ql-symbiflow-run-compile.sh -compile -src work/quicklogic/fpga/fpga_helloworldhw -d ql-eos-s3 -P PU64 -v helloworldfpga.v -t helloworldfpga -p quickfeather.pcf -dump header
+```
+and
+```
+docker-ql-symbiflow-run-compile.sh -compile -src work/quicklogic/fpga/counter_16bit -d ql-eos-s3 -P pd64 -v counter_16bit.v -t top -p counter_16bit.pcf -dump header
+```
+which is equivalent to when using the local installation.
+
+There is more work needed to think about streamlining this process, to make it simpler for an end user to use.
+
+As it is currently, an end user will do the following steps to setup:
+1. clone the repo and cd into this directory
+2. build the docker image using : `docker-ql-symbiflow-build-with-current-user.sh`
+   change the TAG and the docker -t value to a different version as required (currently v1.2.0)
+3. use the `docker-ql-symbiflow-run-compile.sh` to build FPGA designs
+   change the bind mounts as needed
+   currently, it binds ~/work of host to ~/work of the docker container
+   examples:
+`docker-ql-symbiflow-run-compile.sh -compile -src work/quicklogic/fpga/counter_16bit -d ql-eos-s3 -P pd64 -v counter_16bit.v -t top -p counter_16bit.pcf -dump header`
+4. the build output of the toolchain will be directly accessible in the host system, and the user can use these to flash the image etc.
+
+## Test it out (has repetitions from the description)
+You can take the current scripts for a test drive quickly and see the usability and possible improvements.
+1. clone the repo, cd into this dir
+2. build the docker image using : `docker-ql-symbiflow-build-with-current-user.sh`
+3. copy the example fpga designs from `example_fpga_designs` to ~/work/quicklogic/fpga
+   create the dir ~/work/quicklogic/fpga if it doesn't exist
+4. compile the designs using:
+```
+docker-ql-symbiflow-run-compile.sh -compile -src work/quicklogic/fpga/fpga_helloworldhw -d ql-eos-s3 -P PU64 -v helloworldfpga.v -t helloworldfpga -p quickfeather.pcf -dump header
+```
+and
+```
+docker-ql-symbiflow-run-compile.sh -compile -src work/quicklogic/fpga/counter_16bit -d ql-eos-s3 -P pd64 -v counter_16bit.v -t top -p counter_16bit.pcf -dump header
+```
+
+
+## TODO
+1. make the bind mounts easier/more dynamic to use with the help of environment variables
+2. method to use the docker image like above to execute FPGA compilation, equivalent to the usage of a local installation of the ql_symbiflow toolchain in the qorc-sdk as a proof-of-possibility
+3. ensure that the sh scripts can be invoked from other paths as well

more_effective_docker/docker-ql-symbiflow-build-with-current-user.sh

@@ -0,0 +1,11 @@
+# example docker build invocation
+# PW can be set to anything, our runtime config will override this
+
+docker build --build-arg USER=$USER \
+         --build-arg UID=$UID \
+         --build-arg GID=$GID \
+         --build-arg PW=password \
+         --build-arg TAG=v1.2.0 \
+         -t ql_symbiflow_user:v1.2.0 \
+         -f Dockerfile.use-installer-user\
+         .

more_effective_docker/docker-ql-symbiflow-entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+#export INSTALL_DIR="/opt/symbiflow/eos-s3/v1.2.0"
+#export PATH="$INSTALL_DIR/install/bin:$INSTALL_DIR/install/bin/python:$PATH"
+source "$INSTALL_DIR/conda/etc/profile.d/conda.sh"
+conda activate
+
+# execute the help command to display the help
+#ql_symbiflow -h
+
+ql_symbiflow $@

more_effective_docker/docker-ql-symbiflow-run-compile.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+
+# this runs a docker image, replicating the user/group configuration of the host machine
+# this resolves many of the perms settings while using the container.
+# remember to:
+# add mounted volumes for the source code etc.
+# modify the image name to whatever you have on your host.
+
+# -it interactive, tty
+# --rm remove container once it exits, cleanup
+# map the home dir for simplicity, can be removed but keeps it simpler to match container to host. we will revisit this soon.
+# map the group, passwd and shadow files to reflect the user permissions on host system
+# map the work folders as needed
+# set the user in the container as the same one invoking the docker run
+# set the env variable USER (required for android 7.x builds!)
+# set the env variable HOME (not sure if this is needed, but good to define)
+# finally pass in the args given to use, to the container to run $@
+
+docker run -it --rm \
+--volume="/home/$(id -un)/work:/home/$(id -un)/work:rw" \
+--volume="/etc/group:/etc/group:ro" \
+--volume="/etc/passwd:/etc/passwd:ro" \
+--volume="/etc/shadow:/etc/shadow:ro" \
+--user $(id -u):$(id -g) \
+-e USER=$(id -un) \
+-e HOME=/home/$(id -un) \
+--entrypoint=./docker-ql-symbiflow-entrypoint.sh \
+ql_symbiflow_user:v1.2.0 \
+$@	
+
+
+

more_effective_docker/docker-ql-symbiflow-test.sh

@@ -0,0 +1,21 @@
+#export INSTALL_DIR="/opt/symbiflow/eos-s3/v1.2.0"
+#export PATH="$INSTALL_DIR/install/bin:$INSTALL_DIR/install/bin/python:$PATH"
+source "$INSTALL_DIR/conda/etc/profile.d/conda.sh"
+conda activate
+
+# execute the help command to display the help
+ql_symbiflow -h
+
+# by default we are in the ~ dir.
+
+#cd counter_16bit
+ql_symbiflow 	
+#cd -
+
+
+#cd fpga_helloworldhw
+ql_symbiflow -compile -src ~/fpga_helloworldhw -d ql-eos-s3 -P PU64 -v helloworldfpga.v -t helloworldfpga -p quickfeather.pcf -dump header
+#cd -
+
+conda deactivate
+

more_effective_docker/example_fpga_designs/counter_16bit/Makefile.symbiflow

@@ -0,0 +1,47 @@
+.PHONY:${BUILDDIR}
+
+current_dir := /home/coolbreeze413/work/quicklogic/fpga/counter_16bit
+TOP := top
+VERILOG := ${current_dir}/counter_16bit.v  
+PARTNAME := PD64
+DEVICE  := ql-eos-s3_wlcsp
+PCF := ${current_dir}/counter_16bit.pcf
+SDC := ${current_dir}/build/top_dummy.sdc
+BUILDDIR := build
+
+all: ${BUILDDIR}/${TOP}.bit
+
+${BUILDDIR}/${TOP}.eblif: ${VERILOG}
+	cd ${BUILDDIR} && synth -t ${TOP} -v ${VERILOG} -d ${DEVICE} -p ${PCF} -P ${PART} 2>&1 > /home/coolbreeze413/work/quicklogic/fpga/counter_16bit/build/top.log
+
+${BUILDDIR}/${TOP}.net: ${BUILDDIR}/${TOP}.eblif
+	cd ${BUILDDIR} && pack -e ${TOP}.eblif -d ${DEVICE} -s ${SDC} 2>&1 > /home/coolbreeze413/work/quicklogic/fpga/counter_16bit/build/top.log
+
+${BUILDDIR}/${TOP}.place: ${BUILDDIR}/${TOP}.net
+	cd ${BUILDDIR} && place -e ${TOP}.eblif -d ${DEVICE} -p ${PCF} -n ${TOP}.net -P ${PARTNAME} -s ${SDC} 2>&1 > /home/coolbreeze413/work/quicklogic/fpga/counter_16bit/build/top.log
+
+${BUILDDIR}/${TOP}.route: ${BUILDDIR}/${TOP}.place
+	cd ${BUILDDIR} && route -e ${TOP}.eblif -d ${DEVICE} -s ${SDC} 2>&1 > /home/coolbreeze413/work/quicklogic/fpga/counter_16bit/build/top.log
+
+${BUILDDIR}/${TOP}.fasm: ${BUILDDIR}/${TOP}.route
+	cd ${BUILDDIR} && write_fasm -e ${TOP}.eblif -d ${DEVICE}
+
+${BUILDDIR}/${TOP}.bit: ${BUILDDIR}/${TOP}.fasm
+	cd ${BUILDDIR} && write_bitstream -d ${DEVICE} -f ${TOP}.fasm -P ${PARTNAME} -b ${TOP}.bit
+
+${BUILDDIR}/${TOP}_bit.h: ${BUILDDIR}/${TOP}.bit
+	cd ${BUILDDIR} && write_bitheader -b ${TOP}.bit
+
+${BUILDDIR}/${TOP}.jlink: ${BUILDDIR}/${TOP}.bit
+	cd ${BUILDDIR} && write_jlink -f ${TOP}.fasm -P ${PACKAGENAME} -b ${TOP}.bit
+
+${BUILDDIR}/${TOP}_jlink.h: ${BUILDDIR}/${TOP}.jlink
+	cd ${BUILDDIR} && write_jlinkheader
+
+${BUILDDIR}/${TOP}.post_v: ${BUILDDIR}/${TOP}.bit
+	cd ${BUILDDIR} && write_fasm2bels -e ${TOP}.eblif -d ${DEVICE} -p ${PCF} -n ${TOP}.net -P ${PARTNAME}
+	cd ${BUILDDIR} && analysis -e ${TOP}.eblif -d ${DEVICE} -s ${SDC} 2>&1 > /home/coolbreeze413/work/quicklogic/fpga/counter_16bit/build/top.log
+
+clean:
+	rm -rf ${BUILDDIR}
+

more_effective_docker/example_fpga_designs/counter_16bit/counter_16bit.pcf

@@ -0,0 +1,20 @@
+set_io clk A3
+set_io enable C1
+set_io reset A1
+
+set_io count(0) A2
+set_io count(1) B2
+set_io count(2) C3
+set_io count(3) B3
+set_io count(4) B1
+set_io count(5) C4
+set_io count(6) B4
+set_io count(7) A4
+set_io count(8) C5
+set_io count(9) B5
+set_io count(10) D6
+set_io count(11) A5
+set_io count(12) C6
+set_io count(13) E7
+set_io count(14) D7
+set_io count(15) E8

more_effective_docker/example_fpga_designs/counter_16bit/counter_16bit.sdc

@@ -0,0 +1 @@
+create_clock -period 12 -name clk

more_effective_docker/example_fpga_designs/counter_16bit/counter_16bit.v

@@ -0,0 +1,21 @@
+//
+// Copyright (c) 2020 QuickLogic Corporation.  All Rights Reserved.
+//
+// Description :
+//    Example of asimple 16 bit up counter in Verilog HDL
+//
+// Version 1.0 : Initial Creation
+//
+module top (clk, reset, enable, count);
+input clk, reset, enable;
+output [15:0] count;
+reg [15:0] count;                                   
+
+always @ (posedge clk)
+if (reset == 1'b1) begin
+  count <= 0;
+end else if ( enable == 1'b1) begin
+  count <= count + 1;
+end
+
+endmodule