[Refactor] For universal prover task (the update on coordinator)

.github/workflows/coordinator.yml

@@ -112,7 +112,8 @@ jobs:
     - name: Test coordinator packages
       working-directory: 'coordinator'
       run: |
-        # go test -exec "env LD_LIBRARY_PATH=${PWD}/verifier/lib" -v -race -gcflags="-l" -ldflags="-s=false" -coverpkg="scroll-tech/coordinator" -coverprofile=coverage.txt -covermode=atomic ./...
+        make libzkp
+        # go test -exec "env LD_LIBRARY_PATH=${PWD}/../common/libzkp/lib" -v -race -gcflags="-l" -ldflags="-s=false" -coverpkg="scroll-tech/coordinator" -coverprofile=coverage.txt -covermode=atomic ./...
         go test -v -race -gcflags="-l" -ldflags="-s=false" -coverprofile=coverage.txt -covermode=atomic -tags mock_verifier ./...
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v3

.github/workflows/integration.yml

@@ -38,6 +38,7 @@ jobs:
         make dev_docker
         make -C rollup mock_abi
         make -C common/bytecode all
+        make -C common/libzkp/impl build
     - name: Run integration tests
       run: |
         go test -v -tags="mock_prover mock_verifier" -p 1 -coverprofile=coverage.txt scroll-tech/integration-test/...

common/libzkp/go.mod

@@ -0,0 +1,3 @@
+module scroll-tech/common/libzkp
+
+go 1.22

common/libzkp/impl/Cargo.toml

@@ -19,7 +19,6 @@ euclid_verifier = { git = "https://github.com/scroll-tech/zkvm-prover.git", tag
 
 base64 = "0.13.0"
 env_logger = "0.9.0"
-libc = "0.2"
 log = "0.4"
 once_cell = "1.19"
 serde = "1.0"

common/libzkp/impl/Makefile

@@ -2,10 +2,15 @@
 
 build:
 	@cargo build --release
+	@mkdir -p ../lib
+	@cp -f target/release/libzkp.so ../lib/
 
 fmt:
 	@cargo fmt --all -- --check
 
+clean:
+	@rm -f ../lib/libzkp.so
+
 clippy:
 	@cargo check --all-features
 	@cargo clippy --release -- -D warnings

common/libzkp/impl/src/lib.rs

@@ -1,15 +1,17 @@
 mod utils;
 mod verifier;
 
-use std::path::Path;
+use std::{
+    ffi::{c_char, c_int, CString},
+    path::Path,
+};
 
 use crate::utils::{c_char_to_str, c_char_to_vec};
-use libc::c_char;
 use verifier::{TaskType, VerifierConfig};
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn init(config: *const c_char) {
+pub unsafe extern "C" fn init_verifier(config: *const c_char) {
     let config_str = c_char_to_str(config);
     let verifier_config = serde_json::from_str::<VerifierConfig>(config_str).unwrap();
     verifier::init(verifier_config);
@@ -74,3 +76,60 @@ fn _dump_vk(fork_name: *const c_char, file: *const c_char) {
         verifier.as_ref().dump_vk(Path::new(c_char_to_str(file)));
     }
 }
+
+/// Represents the result of generating a universal task
+#[repr(C)]
+pub struct HandlingResult {
+    pub ok: bool,
+    pub universal_task: *mut c_char,
+    pub metadata: *mut c_char,
+    pub expected_pi_hash: [u8; 32],
+}
+
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn gen_universal_task(
+    _task_type: c_int,
+    _task: *const c_char,
+    _fork_name: *const c_char,
+) -> HandlingResult {
+    unimplemented!("implementation will be added in later PRs");
+}
+
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn release_task_result(result: HandlingResult) {
+    // Free the allocated strings
+    if !result.universal_task.is_null() {
+        let _ = CString::from_raw(result.universal_task);
+    }
+
+    if !result.metadata.is_null() {
+        let _ = CString::from_raw(result.metadata);
+    }
+}
+
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn gen_wrapped_proof(
+    _proof_json: *const c_char,
+    _metadata: *const c_char,
+    _vk: *const c_char,
+    _vk_len: usize,
+) -> *mut c_char {
+    unimplemented!("implementation will be added in later PRs");
+}
+
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn release_string(string_ptr: *mut c_char) {
+    if !string_ptr.is_null() {
+        let _ = CString::from_raw(string_ptr);
+    }
+}
+
+/// # Safety
+#[no_mangle]
+pub unsafe extern "C" fn init_l2geth(_config: *const c_char) {
+    unimplemented!("implementation will be added in later PRs");
+}

common/libzkp/impl/src/utils.rs

@@ -1,6 +1,5 @@
 use std::{
-    ffi::CStr,
-    os::raw::c_char,
+    ffi::{c_char, CStr},
     panic::{catch_unwind, AssertUnwindSafe},
 };
 

common/libzkp/interface/libzkp.h

@@ -1,12 +1,48 @@
-// BatchVerifier is used to:
+// Verifier is used to:
 // - Verify a batch proof
 // - Verify a bundle proof
-void init(char* config);
+// - Verify a chunk proof
 
-char verify_batch_proof(char* proof, char* fork_name);
+#ifndef LIBZKP_H
+#define LIBZKP_H
 
-char verify_bundle_proof(char* proof, char* fork_name);
+#include <stddef.h> // For size_t
+
+// Initialize the verifier with configuration
+void init_verifier(char* config);
 
+// Initialize the l2geth with configuration
+void init_l2geth(char* config);
+
+// Verify proofs - returns non-zero for success, zero for failure
+char verify_batch_proof(char* proof, char* fork_name);
+char verify_bundle_proof(char* proof, char* fork_name);
 char verify_chunk_proof(char* proof, char* fork_name);
 
+// Dump verification key to file
 void dump_vk(char* fork_name, char* file);
+
+// The result struct to hold data from handling a proving task
+typedef struct {
+    char ok;
+    char* universal_task;
+    char* metadata;
+    char expected_pi_hash[32];
+} HandlingResult;
+
+// Generate a universal task based on task type and input JSON
+// Returns a struct containing task data, metadata, and expected proof hash
+HandlingResult gen_universal_task(int task_type, char* task, char* fork_name);
+
+// Release memory allocated for a HandlingResult returned by gen_universal_task
+void release_task_result(HandlingResult result);
+
+// Generate a wrapped proof from the universal prover output and metadata
+// Returns a JSON string containing the wrapped proof, or NULL on error
+// The caller must call release_string on the returned pointer when done
+char* gen_wrapped_proof(char* proof_json, char* metadata, char* vk, size_t vk_len);
+
+// Release memory allocated for a string returned by gen_wrapped_proof
+void release_string(char* string_ptr);
+
+#endif /* LIBZKP_H */

common/libzkp/lib.go

@@ -0,0 +1,143 @@
+package libzkp
+
+/*
+#cgo LDFLAGS: -lzkp -lm -ldl -L${SRCDIR}/lib -Wl,-rpath=${SRCDIR}/lib
+#cgo gpu LDFLAGS: -lzkp -lm -ldl -lgmp -lstdc++ -lprocps -L/usr/local/cuda/lib64/ -lcudart -L${SRCDIR}/lib/ -Wl,-rpath=${SRCDIR}/lib
+#include <stdlib.h>
+#include "interface/libzkp.h"
+*/
+import "C" //nolint:typecheck
+
+import (
+	"fmt"
+	"os"
+	"unsafe"
+)
+
+// Helper function to convert Go string to C string and handle cleanup
+func goToCString(s string) *C.char {
+	return C.CString(s)
+}
+
+// Helper function to free C string
+func freeCString(s *C.char) {
+	C.free(unsafe.Pointer(s))
+}
+
+// Initialize the verifier
+func InitVerifier(configJSON string) {
+	cConfig := goToCString(configJSON)
+	defer freeCString(cConfig)
+
+	C.init_verifier(cConfig)
+}
+
+// Initialize the verifier
+func InitL2geth(configJSON string) {
+	cConfig := goToCString(configJSON)
+	defer freeCString(cConfig)
+
+	C.init_l2geth(cConfig)
+}
+
+// Verify a chunk proof
+func VerifyChunkProof(proofData, forkName string) bool {
+	cProof := goToCString(proofData)
+	cForkName := goToCString(forkName)
+	defer freeCString(cProof)
+	defer freeCString(cForkName)
+
+	result := C.verify_chunk_proof(cProof, cForkName)
+	return result != 0
+}
+
+// Verify a batch proof
+func VerifyBatchProof(proofData, forkName string) bool {
+	cProof := goToCString(proofData)
+	cForkName := goToCString(forkName)
+	defer freeCString(cProof)
+	defer freeCString(cForkName)
+
+	result := C.verify_batch_proof(cProof, cForkName)
+	return result != 0
+}
+
+// Verify a bundle proof
+func VerifyBundleProof(proofData, forkName string) bool {
+	cProof := goToCString(proofData)
+	cForkName := goToCString(forkName)
+	defer freeCString(cProof)
+	defer freeCString(cForkName)
+
+	result := C.verify_bundle_proof(cProof, cForkName)
+	return result != 0
+}
+
+// Generate a universal task
+func GenerateUniversalTask(taskType int, taskJSON, forkName string) (bool, string, string, []byte) {
+	cTask := goToCString(taskJSON)
+	cForkName := goToCString(forkName)
+	defer freeCString(cTask)
+	defer freeCString(cForkName)
+
+	result := C.gen_universal_task(C.int(taskType), cTask, cForkName)
+	defer C.release_task_result(result)
+
+	// Check if the operation was successful
+	if result.ok == 0 {
+		return false, "", "", nil
+	}
+
+	// Convert C strings to Go strings
+	universalTask := C.GoString(result.universal_task)
+	metadata := C.GoString(result.metadata)
+
+	// Convert C array to Go slice
+	piHash := make([]byte, 32)
+	for i := 0; i < 32; i++ {
+		piHash[i] = byte(result.expected_pi_hash[i])
+	}
+
+	return true, universalTask, metadata, piHash
+}
+
+// Generate wrapped proof
+func GenerateWrappedProof(proofJSON, metadata string, vkData []byte) string {
+	cProofJSON := goToCString(proofJSON)
+	cMetadata := goToCString(metadata)
+	defer freeCString(cProofJSON)
+	defer freeCString(cMetadata)
+
+	// Create a C array from Go slice
+	cVkData := (*C.char)(unsafe.Pointer(&vkData[0]))
+
+	resultPtr := C.gen_wrapped_proof(cProofJSON, cMetadata, cVkData, C.size_t(len(vkData)))
+	if resultPtr == nil {
+		return ""
+	}
+
+	// Convert result to Go string and free C memory
+	result := C.GoString(resultPtr)
+	C.release_string(resultPtr)
+
+	return result
+}
+
+// Dumps a verification key to a file
+func DumpVk(forkName, filePath string) error {
+	cForkName := goToCString(forkName)
+	cFilePath := goToCString(filePath)
+	defer freeCString(cForkName)
+	defer freeCString(cFilePath)
+
+	// Call the C function to dump the verification key
+	C.dump_vk(cForkName, cFilePath)
+
+	// Check if the file was created successfully
+	// Note: The C function doesn't return an error code, so we check if the file exists
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return fmt.Errorf("failed to dump verification key: file %s was not created", filePath)
+	}
+
+	return nil
+}

coordinator/.gitignore

@@ -1,3 +1,4 @@
 /build/bin
 .idea
 internal/logic/verifier/lib
+internal/libzkp/lib/libzkp.so

coordinator/Makefile

@@ -16,11 +16,15 @@ ZK_VERSION=${ZKEVM_VERSION}-${HALO2_VERSION}
 test:
 	go test -v -race -coverprofile=coverage.txt -covermode=atomic -p 1 $(PWD)/...
 
-libzkp:
-	cd ../common/libzkp/impl && cargo clean && cargo build --release && cp ./target/release/libzkp.so ../interface/
-	rm -rf ./internal/logic/verifier/lib && cp -r ../common/libzkp/interface ./internal/logic/verifier/lib
+../common/libzkp/lib/libzkp.so:
+	$(MAKE) -C ../common/libzkp/impl build
 
-coordinator_api: libzkp ## Builds the Coordinator api instance.
+clean_libzkp:
+	$(MAKE) -C ../common/libzkp/impl clean
+
+libzkp: clean_libzkp ../common/libzkp/lib/libzkp.so
+
+coordinator_api: ../common/libzkp/lib/libzkp.so ## Builds the Coordinator api instance.
 	go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_api ./cmd/api
 
 coordinator_cron:
@@ -38,14 +42,14 @@ mock_coordinator_api: ## Builds the mocked Coordinator instance.
 mock_coordinator_cron: ## Builds the mocked Coordinator instance.
 	go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator_cron ./cmd/cron
 
-test-verifier: libzkp
+test-verifier: ../common/libzkp/lib/libzkp.so
 	go test -tags ffi -timeout 0 -v ./internal/logic/verifier
 
-test-gpu-verifier: libzkp
+test-gpu-verifier: ../common/libzkp/lib/libzkp.so
 	go test -tags="gpu ffi" -timeout 0 -v ./internal/logic/verifier
 
 lint: ## Lint the files - used for CI
-	cp -r ../common/libzkp/interface ./internal/logic/verifier/lib
+#	cp -r ../common/libzkp/interface ./internal/logic/verifier/lib
 	GOBIN=$(PWD)/build/bin go run ../build/lint.go
 
 clean: ## Empty out the bin folder