diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..42d0284 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +FROM python:3.12-slim AS build + +# Install build dependencies and UV +RUN apt-get update && \ + apt-get install -y build-essential curl && \ + pip install --no-cache-dir uv && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy only requirements file first to leverage Docker cache +COPY requirements.txt* ./ + +# Install dependencies using UV for faster installation +RUN if [ -f "requirements.txt" ]; then \ + uv pip install --system -r requirements.txt; \ + fi + +# Final stage +FROM python:3.12-slim + +# Accept module name and port as build arguments +ARG MODULE_NAME +ARG PORT=8002 + +# Set environment variables from build args +ENV PORT=$PORT +ENV MODULE_NAME=hackmd_sensor_node + +WORKDIR /app + +# Install only runtime dependencies (curl for healthcheck) +RUN apt-get update && \ + apt-get install -y curl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Copy installed dependencies from build stage +COPY --from=build /usr/local/lib/python3.12/site-packages/ /usr/local/lib/python3.12/site-packages/ +COPY --from=build /usr/local/bin/ /usr/local/bin/ + +# Copy project files and source code +COPY . /app/ + +# Expose port for container +EXPOSE $PORT + +# Configure healthcheck +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ + CMD curl --fail http://localhost:$PORT/koi-net/health || exit 1 + +# Start server using environment variables +# The module name is used to determine which server module to load +CMD uvicorn hackmd_sensor_node.server:app --host 0.0.0.0 --port $PORT diff --git a/hackmd_sensor_node/__init__.py b/hackmd_sensor_node/__init__.py index 8862c67..6827c21 100644 --- a/hackmd_sensor_node/__init__.py +++ b/hackmd_sensor_node/__init__.py @@ -1,23 +1,31 @@ import logging from rich.logging import RichHandler +log_level_str = "DEBUG" + logger = logging.getLogger() -logger.setLevel(logging.DEBUG) +logger.setLevel(log_level_str.upper()) + +# Remove existing handlers to avoid duplicates if this module is reloaded +for handler in logger.handlers[:]: + logger.removeHandler(handler) -rich_handler = RichHandler() -rich_handler.setLevel(logging.INFO) +# Use stderr=True if you want logs to go to stderr instead of stdout +rich_handler = RichHandler(rich_tracebacks=True, show_path=False, log_time_format="%Y-%m-%d %H:%M:%S") +rich_handler.setLevel(log_level_str.upper()) # Set level for this handler rich_handler.setFormatter(logging.Formatter( - "%(name)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" + fmt="%(name)s - %(message)s", # Simplified format for console + datefmt="[%X]" # Use RichHandler's default time format )) +logger.addHandler(rich_handler) -file_handler = logging.FileHandler("node-log.txt") -file_handler.setLevel(logging.DEBUG) +# Add file handler to write logs to node.sensor.log +file_handler = logging.FileHandler("node.sensor.log") +file_handler.setLevel(log_level_str.upper()) file_handler.setFormatter(logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s", + fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S" )) +logger.addHandler(file_handler) -# Add both -logger.addHandler(rich_handler) -logger.addHandler(file_handler) \ No newline at end of file +logger.info(f"Logging initialized for HackMD Sensor Node. Level: {log_level_str.upper()}.") diff --git a/hackmd_sensor_node/__main__.py b/hackmd_sensor_node/__main__.py index 06bb10d..f99f8e1 100644 --- a/hackmd_sensor_node/__main__.py +++ b/hackmd_sensor_node/__main__.py @@ -1,8 +1,19 @@ import uvicorn +import logging from .core import node -uvicorn.run( - "hackmd_sensor_node.server:app", - host=node.config.server.host, - port=node.config.server.port, -) \ No newline at end of file +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if not node.config.server or not node.config.server.host or node.config.server.port is None: + logger.critical("Server configuration (host/port) is missing in the loaded config. Cannot start.") + exit(1) + + logger.info(f"Starting HackMD sensor node server on {node.config.server.host}:{node.config.server.port}") + uvicorn.run( + "hackmd_sensor_node.server:app", + host=node.config.server.host, + port=node.config.server.port, + log_config=None, + reload=False + ) \ No newline at end of file diff --git a/hackmd_sensor_node/backfill.py b/hackmd_sensor_node/backfill.py index c230c40..ee2830b 100644 --- a/hackmd_sensor_node/backfill.py +++ b/hackmd_sensor_node/backfill.py @@ -1,30 +1,183 @@ import logging -import asyncio from rid_lib.ext import Bundle from rid_types import HackMDNote from . import hackmd_api from .core import node +from .config import StateType logger = logging.getLogger(__name__) -async def backfill(team_path="blockscience"): - notes = await hackmd_api.async_request(f"/teams/{team_path}/notes") - - logger.debug(f"Found {len(notes)} in team") - - for note in notes: - note_rid = HackMDNote(note["id"]) - - note_bundle = Bundle.generate( - rid=note_rid, - contents=note - ) - - node.processor.handle(bundle=note_bundle) - -if __name__ == "__main__": - node.start() - asyncio.run( - backfill() - ) - node.stop() \ No newline at end of file +async def _process_team_notes(team_path, state): + processed_count = 0 + bundled_count = 0 + logger.info(f"Processing all notes in team path: '{team_path}'") + if not team_path: + logger.error("HackMD team path is not configured. Cannot backfill all team notes.") + return processed_count, bundled_count + team_notes = await hackmd_api.async_request(f"/teams/{team_path}/notes") + if not team_notes: + logger.warning(f"No notes found or error fetching notes for team '{team_path}'. Backfill ending.") + return processed_count, bundled_count + logger.debug(f"Found {len(team_notes)} notes in team summary.") + for note_summary in team_notes: + processed_count += 1 + note_id = note_summary.get("id") + last_modified_str = note_summary.get("lastChangedAt") + title = note_summary.get("title", f"Note {note_id}") + if not note_id or not last_modified_str: + logger.warning(f"Skipping note from team list due to missing ID or lastChangedAt: {note_summary}") + continue + if note_id not in state or last_modified_str > state[note_id]: + logger.info(f"Processing note '{title}' (ID: {note_id}) from team list - New or updated.") + note_details = hackmd_api.request(f"/notes/{note_id}") + if not note_details: + logger.error(f"Failed to fetch details for note ID {note_id} from team list. Skipping.") + continue + try: + rid = HackMDNote(note_id=note_id) + contents = { + "id": note_id, + "title": title, + "content": note_details.get("content"), + "createdAt": note_details.get("createdAt"), + "lastChangedAt": note_details.get("lastChangedAt", last_modified_str), + "publishLink": note_details.get("publishLink"), + "tags": note_details.get("tags", []), + } + if contents["content"] is None: + logger.error(f"Content missing for note ID {note_id} from team list. Skipping bundle.") + continue + bundle = Bundle.generate(rid=rid, contents=contents) + logger.debug(f"Making backfill note bundle {rid} from team list available locally.") + node.processor.handle(bundle=bundle) + bundled_count += 1 + state[note_id] = contents["lastChangedAt"] + except Exception as e: + logger.error(f"Error creating/handling bundle for note {note_id} from team list: {e}", exc_info=True) + else: + logger.debug(f"Skipping note '{title}' (ID: {note_id}) from team list - Already up-to-date.") + return processed_count, bundled_count + +def _process_target_notes(target_note_ids, state): + processed_count = 0 + bundled_count = 0 + logger.info(f"Targeting specific HackMD notes for backfill: {target_note_ids}") + for note_id in target_note_ids: + processed_count += 1 + logger.debug(f"Fetching targeted note ID: {note_id}") + note_details = hackmd_api.request(f"/notes/{note_id}") + if not note_details: + logger.warning(f"Could not fetch details for targeted note ID {note_id}. Skipping.") + continue + last_modified_str = note_details.get("lastChangedAt") + title = note_details.get("title", f"Note {note_id}") + if not last_modified_str: + logger.warning(f"Skipping targeted note {note_id} ('{title}') due to missing lastChangedAt.") + continue + if note_id not in state or last_modified_str > state[note_id]: + logger.info(f"Processing targeted note '{title}' (ID: {note_id}) - New or updated.") + try: + rid = HackMDNote(note_id=note_id) + contents = { + "id": note_id, + "title": title, + "content": note_details.get("content"), + "createdAt": note_details.get("createdAt"), + "lastChangedAt": last_modified_str, + "publishLink": note_details.get("publishLink"), + "tags": note_details.get("tags", []), + } + if contents["content"] is None: + logger.error(f"Content missing for targeted note ID {note_id}. Skipping bundle.") + continue + bundle = Bundle.generate(rid=rid, contents=contents) + logger.debug(f"Making backfill targeted note bundle {rid} available locally.") + node.processor.handle(bundle=bundle) + bundled_count += 1 + state[note_id] = last_modified_str + except Exception as e: + logger.error(f"Error bundling targeted note {note_id}: {e}", exc_info=True) + else: + logger.debug(f"Skipping targeted note '{title}' (ID: {note_id}) - Already up-to-date.") + return processed_count, bundled_count + + +async def backfill(state: StateType): + """Fetches notes, compares with state, and bundles new/updated notes.""" + + team_path = getattr(node.config.hackmd, 'team_path', "blockscience") # Safer access with default + target_note_ids = getattr(node.config.hackmd, 'target_note_ids', None) + + logger.info("Starting HackMD backfill.") + + try: + processed_count = 0 + bundled_count = 0 + + # Decide whether to process specific notes or all team notes + if target_note_ids: + pc, bc = _process_target_notes(target_note_ids, state) + processed_count += pc + bundled_count += bc + else: + # Original logic: process all notes in the team + logger.info(f"Processing all notes in team path: \'{team_path}\'") + if not team_path: + logger.error("HackMD team path is not configured. Cannot backfill all team notes.") + return + + team_notes = await hackmd_api.async_request(f"/teams/{team_path}/notes") + if not team_notes: + logger.warning(f"No notes found or error fetching notes for team \'{team_path}\'. Backfill ending.") + return + + logger.debug(f"Found {len(team_notes)} notes in team summary.") + for note_summary in team_notes: + processed_count += 1 + note_id = note_summary.get("id") + last_modified_str = note_summary.get("lastChangedAt") + title = note_summary.get("title", f"Note {note_id}") + + if not note_id or not last_modified_str: + logger.warning(f"Skipping note from team list due to missing ID or lastChangedAt: {note_summary}") + continue + + # Check if note needs processing based on state + if note_id not in state or last_modified_str > state[note_id]: + logger.info(f"Processing note \'{title}\' (ID: {note_id}) from team list - New or updated.") + # Fetch full content only when needed + note_details = hackmd_api.request(f"/notes/{note_id}") + if not note_details: + logger.error(f"Failed to fetch details for note ID {note_id} from team list. Skipping.") + continue + + try: + rid = HackMDNote(note_id=note_id) + contents = { + "id": note_id, + "title": title, + "content": note_details.get("content"), + "createdAt": note_details.get("createdAt"), + "lastChangedAt": note_details.get("lastChangedAt", last_modified_str), + "publishLink": note_details.get("publishLink"), + "tags": note_details.get("tags", []), + } + if contents["content"] is None: + logger.error(f"Content missing for note ID {note_id} from team list. Skipping bundle.") + continue + + bundle = Bundle.generate(rid=rid, contents=contents) + logger.debug(f"Making backfill note bundle {rid} from team list available locally.") + node.processor.handle(bundle=bundle) + bundled_count += 1 + state[note_id] = contents["lastChangedAt"] # Update state with timestamp used + + except Exception as e: + logger.error(f"Error creating/handling bundle for note {note_id} from team list: {e}", exc_info=True) + else: + logger.debug(f"Skipping note \'{title}\' (ID: {note_id}) from team list - Already up-to-date.") + + logger.info(f"HackMD backfill complete. Processed {processed_count} notes, bundled {bundled_count} new/updated notes.") + + except Exception as e: + logger.error(f"Unexpected error during HackMD backfill: {e}", exc_info=True) diff --git a/hackmd_sensor_node/config.py b/hackmd_sensor_node/config.py index bd780db..94c8000 100644 --- a/hackmd_sensor_node/config.py +++ b/hackmd_sensor_node/config.py @@ -1,16 +1,28 @@ +import json +import logging from pydantic import BaseModel, Field from koi_net.protocol.node import NodeProfile, NodeType, NodeProvides from koi_net.config import NodeConfig, EnvConfig, KoiNetConfig from rid_types import HackMDNote +from pathlib import Path +from typing import Dict + +# Define StateType here to avoid circular import +StateType = Dict[str, str] + +logger = logging.getLogger(__name__) + +HACKMD_STATE_FILE_PATH = Path(".koi/hackmd/hackmd_state.json") class HackMDConfig(BaseModel): team_path: str | None = "blockscience" + target_note_ids: list[str] | None = None class HackMDEnvConfig(EnvConfig): hackmd_api_token: str | None = "HACKMD_API_TOKEN" class HackMDSensorNodeConfig(NodeConfig): - koi_net: KoiNetConfig | None = Field(default_factory = lambda: + koi_net: KoiNetConfig = Field(default_factory=lambda: KoiNetConfig( node_name="hackmd-sensor", node_profile=NodeProfile( @@ -23,4 +35,31 @@ class HackMDSensorNodeConfig(NodeConfig): ) ) env: HackMDEnvConfig | None = Field(default_factory=HackMDEnvConfig) - hackmd: HackMDConfig | None = Field(default_factory=HackMDConfig) \ No newline at end of file + hackmd: HackMDConfig | None = Field(default_factory=HackMDConfig) + +# --- State Management Functions --- +def load_hackmd_state() -> StateType: + """Loads the last modified timestamp state from the JSON file.""" + try: + HACKMD_STATE_FILE_PATH.parent.mkdir(parents=True, exist_ok=True) + if HACKMD_STATE_FILE_PATH.exists(): + with open(HACKMD_STATE_FILE_PATH, "r") as f: + state_data = json.load(f) + logger.info(f"Loaded HackMD state from '{HACKMD_STATE_FILE_PATH}': {len(state_data)} notes tracked.") + return state_data + else: + logger.info(f"HackMD state file '{HACKMD_STATE_FILE_PATH}' not found. Starting empty.") + return {} + except Exception as e: + logger.error(f"Error loading HackMD state file '{HACKMD_STATE_FILE_PATH}': {e}", exc_info=True) + return {} + +def save_hackmd_state(state: StateType): + """Saves the state dictionary to the JSON file.""" + try: + HACKMD_STATE_FILE_PATH.parent.mkdir(parents=True, exist_ok=True) + with open(HACKMD_STATE_FILE_PATH, "w") as f: + json.dump(state, f, indent=4) + logger.debug(f"Saved HackMD state to '{HACKMD_STATE_FILE_PATH}'.") + except Exception as e: + logger.error(f"Error writing HackMD state file '{HACKMD_STATE_FILE_PATH}': {e}", exc_info=True) diff --git a/hackmd_sensor_node/core.py b/hackmd_sensor_node/core.py index 7965288..2a61c4d 100644 --- a/hackmd_sensor_node/core.py +++ b/hackmd_sensor_node/core.py @@ -3,21 +3,20 @@ from koi_net.processor.default_handlers import ( basic_rid_handler, edge_negotiation_handler, - basic_network_output_filter + basic_network_output_filter, + basic_manifest_handler # Add this import ) from .config import HackMDSensorNodeConfig logger = logging.getLogger(__name__) - node = NodeInterface( config=HackMDSensorNodeConfig.load_from_yaml("config.yaml"), use_kobj_processor_thread=True, handlers=[ basic_rid_handler, edge_negotiation_handler, - basic_network_output_filter + basic_network_output_filter, + basic_manifest_handler, ] ) - -from . import handlers \ No newline at end of file diff --git a/hackmd_sensor_node/handlers.py b/hackmd_sensor_node/handlers.py index 2a92bc8..9e8d66e 100644 --- a/hackmd_sensor_node/handlers.py +++ b/hackmd_sensor_node/handlers.py @@ -3,11 +3,11 @@ from koi_net.processor.knowledge_object import KnowledgeSource, KnowledgeObject from koi_net.processor.interface import ProcessorInterface from koi_net.protocol.event import EventType -from koi_net.protocol.edge import EdgeType +from koi_net.protocol.edge import EdgeType, EdgeProfile, EdgeStatus from koi_net.protocol.node import NodeProfile from koi_net.protocol.helpers import generate_edge_bundle from rid_lib.ext import Bundle -from rid_lib.types import KoiNetNode +from rid_lib.types import KoiNetNode, KoiNetEdge from rid_types import HackMDNote from .core import node @@ -28,10 +28,10 @@ def coordinator_contact(processor: ProcessorInterface, kobj: KnowledgeObject): if KoiNetNode not in node_profile.provides.event: return - logger.debug("Identified a coordinator!") - logger.debug("Proposing new edge") + logger.info("Identified a coordinator for network discovery!") + logger.info("Proposing bidirectional edges for network discovery...") - # queued for processing + # First edge proposal - FROM Coordinator TO Sensor (existing) processor.handle(bundle=generate_edge_bundle( source=kobj.rid, target=node.identity.rid, @@ -39,7 +39,17 @@ def coordinator_contact(processor: ProcessorInterface, kobj: KnowledgeObject): rid_types=[KoiNetNode] )) - logger.debug("Catching up on network state") + # Second edge proposal - FROM Sensor TO Coordinator (critical for discovery) + processor.handle(bundle=generate_edge_bundle( + source=node.identity.rid, + target=kobj.rid, + edge_type=EdgeType.WEBHOOK, + rid_types=[KoiNetNode, HackMDNote] # Include both KoiNetNode and HackMDNote + )) + + logger.info(f"Proposed two edges for bidirectional communication with Coordinator {kobj.rid}") + + logger.info("NETWORK SETUP: Catching up on network state...") rid_payload = processor.network.request_handler.fetch_rids(kobj.rid, rid_types=[KoiNetNode]) @@ -49,20 +59,23 @@ def coordinator_contact(processor: ProcessorInterface, kobj: KnowledgeObject): not processor.cache.exists(rid) ] + logger.info(f"NETWORK SETUP: Found {len(rids)} network nodes to fetch from coordinator") + bundle_payload = processor.network.request_handler.fetch_bundles(kobj.rid, rids=rids) for bundle in bundle_payload.bundles: # marked as external since we are handling RIDs from another node # will fetch remotely instead of checking local cache + logger.info(f"NETWORK SETUP: Processing network node bundle for {bundle.rid}") processor.handle(bundle=bundle, source=KnowledgeSource.External) - logger.debug("Done") + logger.info("NETWORK SETUP: Network initialization completed") @node.processor.register_handler(HandlerType.Manifest) def custom_manifest_handler(processor: ProcessorInterface, kobj: KnowledgeObject): if type(kobj.rid) == HackMDNote: logger.debug("Skipping HackMD note manifest handling") - return + return kobj prev_bundle = processor.cache.read(kobj.rid) @@ -87,7 +100,15 @@ def custom_manifest_handler(processor: ProcessorInterface, kobj: KnowledgeObject @node.processor.register_handler(HandlerType.Bundle, rid_types=[HackMDNote]) def custom_hackmd_bundle_handler(processor: ProcessorInterface, kobj: KnowledgeObject): assert type(kobj.rid) == HackMDNote - + + # Guard against missing summary keys + if 'lastChangedAt' not in kobj.contents: + logger.error(f"Bundle missing 'lastChangedAt' for RID {kobj.rid}. Aborting.") + return STOP_CHAIN + if 'content' not in kobj.contents or kobj.contents.get('content') is None: + logger.error(f"Bundle missing or empty 'content' for RID {kobj.rid}. Aborting.") + return STOP_CHAIN + prev_bundle = processor.cache.read(kobj.rid) if prev_bundle: @@ -123,4 +144,58 @@ def custom_hackmd_bundle_handler(processor: ProcessorInterface, kobj: KnowledgeO kobj.manifest = full_note_bundle.manifest kobj.contents = full_note_bundle.contents - return kobj \ No newline at end of file + return kobj + + +@node.processor.register_handler( + handler_type=HandlerType.Bundle, + rid_types=[KoiNetEdge], + source=KnowledgeSource.External, + event_types=[EventType.NEW] +) +def handle_incoming_edge_proposal(processor: ProcessorInterface, kobj: KnowledgeObject): + logger.info(f"Sensor: BH - Received NEW KoiNetEdge proposal {kobj.rid}") + + if not kobj.contents: + logger.error(f"Sensor: BH - KoiNetEdge KObj {kobj.rid} has no contents even at Bundle stage. This is unexpected. KObj: {kobj!r}") + return STOP_CHAIN + + try: + edge_profile = EdgeProfile.model_validate(kobj.contents) + except Exception as e: + logger.error(f"Sensor: BH - Error validating EdgeProfile for {kobj.rid}: {e}. KObj: {kobj!r}", exc_info=True) + return STOP_CHAIN + + # Automatically accepting all edge proposals for HackMDNote subscriptions + if HackMDNote in edge_profile.rid_types: + logger.info(f"Sensor: BH - Automatically approving edge {edge_profile.source} -> {edge_profile.target} for HackMDNote") + + # Log network graph state for debugging + logger.info(f"NETWORK GRAPH STATE: Known nodes: {[n for n in processor.network_graph.nodes]}") + + # Add small delay to give network discovery time to complete + import time + logger.info(f"DELAY: Waiting 2 seconds before approval to allow network graph to update...") + time.sleep(2) + + edge_profile.status = EdgeStatus.APPROVED + + # Re-queue this approved edge as an INTERNAL UPDATE event + update_bundle = Bundle( + rid=kobj.rid, + contents=edge_profile.model_dump(), + timestamp=kobj.bundle.timestamp, + ) + + processor.handle(bundle=update_bundle, source=KnowledgeSource.Internal, event_type=EventType.UPDATE) + + logger.info(f"Sensor: BH - Requeued APPROVED edge {kobj.rid} with source={KnowledgeSource.Internal}") + + # Check if we have the required nodes in the network graph + target_node = edge_profile.source + if target_node not in processor.network_graph.nodes: + logger.warning(f"Sensor: BH - IMPORTANT: Target node {target_node} is not in our network graph!") + + return STOP_CHAIN + + return diff --git a/hackmd_sensor_node/server.py b/hackmd_sensor_node/server.py index f57fd9e..e324aae 100644 --- a/hackmd_sensor_node/server.py +++ b/hackmd_sensor_node/server.py @@ -3,6 +3,7 @@ from contextlib import asynccontextmanager from fastapi import FastAPI, APIRouter from koi_net.processor.knowledge_object import KnowledgeSource +from rid_lib.types import KoiNetNode from koi_net.protocol.api_models import ( PollEvents, FetchRids, @@ -20,34 +21,136 @@ FETCH_MANIFESTS_PATH, FETCH_BUNDLES_PATH ) + from .core import node from .backfill import backfill +from .config import load_hackmd_state, save_hackmd_state +from . import hackmd_api logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) - +# --- Updated Backfill Loop --- async def backfill_loop(): + # Determine sleep duration (e.g., from config or default) + sleep_duration = 600 # Example: 10 minutes + # sleep_duration = getattr(node.config.hackmd, 'backfill_interval_seconds', 600) # If added to config + + logger.info(f"HackMD backfill loop starting. Interval: {sleep_duration} seconds.") + # Load state ONCE before the loop starts continuous updates + current_state = load_hackmd_state() while True: - await backfill() - await asyncio.sleep(600) + try: + logger.info("Starting periodic HackMD backfill...") + # Pass the current state map to the backfill function + await backfill(current_state) + # Save the potentially modified state after backfill completes + save_hackmd_state(current_state) + logger.info("Periodic HackMD backfill completed.") + except Exception as e: + logger.error(f"Error during periodic HackMD backfill: {e}", exc_info=True) + await asyncio.sleep(sleep_duration) +# --- Updated Lifespan Context Manager --- @asynccontextmanager -async def lifespan(app: FastAPI): - node.start() - asyncio.create_task( - backfill_loop() - ) - - yield - node.stop() +async def lifespan(app: FastAPI): + logger.info("HackMD Sensor Node: FastAPI application startup...") + try: + node.start() + logger.info(f"KOI-net node {node.identity.rid} started.") + + # Network initialization sequence + logger.info("NETWORK SETUP: Beginning network initialization sequence...") + + # Wait a moment before initializing connections + await asyncio.sleep(1) + + # Explicitly initiate coordinator contact if configured + if hasattr(node.config.koi_net, 'first_contact') and node.config.koi_net.first_contact: + logger.info(f"NETWORK SETUP: Initiating first contact with coordinator: {node.config.koi_net.first_contact}") + try: + # Fetch the coordinator's bundle with retries + coordinator_bundles = None + retry_delay = 1 + for attempt in range(1, 4): + try: + coordinator_bundles = node.network.request_handler.fetch_bundles( + url=node.config.koi_net.first_contact, + rids=[KoiNetNode.generate("coordinator")] + ) + break + except Exception as e: + if attempt < 4: + logger.warning(f"NETWORK SETUP: Attempt {attempt} to fetch coordinator failed: {e}. Retrying in {retry_delay}s.") + await asyncio.sleep(retry_delay) + retry_delay *= 2 + else: + logger.error(f"NETWORK SETUP: Failed to fetch coordinator after {attempt} attempts: {e}") + + if coordinator_bundles.bundles: + coordinator_bundle = coordinator_bundles.bundles[0] + logger.info(f"NETWORK SETUP: Received coordinator bundle: {coordinator_bundle.rid}. Queueing for processing.") + node.processor.handle( + bundle=coordinator_bundle, + source=KnowledgeSource.External + ) + + # Give some time for the network to establish + await asyncio.sleep(2) + + # Log network state + known_nodes = node.network.graph.list_nodes() + logger.info(f"NETWORK SETUP: Known nodes after initialization: {known_nodes}") + + current_edges = node.network.graph.list_edges() + logger.info(f"NETWORK SETUP: Current edges after initialization: {current_edges}") + else: + logger.warning("NETWORK SETUP: Could not fetch coordinator bundle") + except Exception as e: + logger.error(f"NETWORK SETUP: Error during network initialization: {e}") + import traceback + logger.debug(traceback.format_exc()) + + logger.info("NETWORK SETUP: Network initialization complete") + + # Initial backfill on startup (optional) + logger.info("Performing initial HackMD backfill on startup...") + initial_state = load_hackmd_state() + # Run initial backfill - pass the state, save after it finishes + await backfill(initial_state) + save_hackmd_state(initial_state) + logger.info("Initial HackMD backfill completed.") + + # Start the periodic backfill loop + asyncio.create_task(backfill_loop()) # Loop manages its own state loading/saving now + + except Exception as e: + logger.critical(f"Critical error during HackMD node startup: {e}", exc_info=True) + raise RuntimeError("Failed to initialize KOI-net node components") from e + + yield # Application runs here + + logger.info("HackMD Sensor Node: FastAPI application shutdown...") + try: + node.stop() + logger.info(f"KOI-net node {node.identity.rid} stopped.") + except Exception as e: + logger.error(f"Error stopping KOI-net node: {e}", exc_info=True) + logger.info("HackMD Sensor Node: Shutdown complete.") + app = FastAPI( - lifespan=lifespan, + lifespan=lifespan, title="KOI-net Protocol API", version="1.0.0" ) +@app.get("/health", tags=["System"]) +async def health_check(): + """Basic health check for the service.""" + return {"status": "healthy", "node_id": str(node.identity.rid) if node.identity else "uninitialized"} + koi_net_router = APIRouter( prefix="/koi-net" @@ -59,7 +162,7 @@ def broadcast_events(req: EventsPayload): for event in req.events: logger.info(f"{event!r}") node.processor.handle(event=event, source=KnowledgeSource.External) - + @koi_net_router.post(POLL_EVENTS_PATH) def poll_events(req: PollEvents) -> EventsPayload: @@ -79,5 +182,4 @@ def fetch_manifests(req: FetchManifests) -> ManifestsPayload: def fetch_bundles(req: FetchBundles) -> BundlesPayload: return node.network.response_handler.fetch_bundles(req) - -app.include_router(koi_net_router) \ No newline at end of file +app.include_router(koi_net_router) diff --git a/node.sensor.log b/node.sensor.log new file mode 100644 index 0000000..75afdf1 --- /dev/null +++ b/node.sensor.log @@ -0,0 +1,146 @@ +2025-05-12 23:35:12 - root - INFO - Logging initialized for HackMD Sensor Node. Level: DEBUG. +2025-05-12 23:35:13 - __main__ - INFO - Starting HackMD sensor node server on 127.0.0.1:8002 +2025-05-12 23:35:13 - asyncio - DEBUG - Using selector: KqueueSelector +2025-05-12 23:35:13 - uvicorn.error - INFO - Started server process [94462] +2025-05-12 23:35:13 - uvicorn.error - INFO - Waiting for application startup. +2025-05-12 23:35:13 - hackmd_sensor_node.server - INFO - HackMD Sensor Node: FastAPI application startup... +2025-05-12 23:35:13 - koi_net.core - INFO - Starting processor worker thread +2025-05-12 23:35:13 - koi_net.network.graph - DEBUG - Generating network graph +2025-05-12 23:35:13 - koi_net.network.graph - DEBUG - Done +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Queued 'None', source: 'INTERNAL'> +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Dequeued 'None', source: 'INTERNAL'> +2025-05-12 23:35:13 - koi_net.core - DEBUG - Waiting for kobj queue to empty +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Handling 'None', source: 'INTERNAL'> +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Calling rid handler 'basic_rid_handler' +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Calling manifest handler 'basic_manifest_handler' +2025-05-12 23:35:13 - koi_net.processor.default_handlers - DEBUG - RID previously unknown to me, labeling as 'NEW' +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Knowledge object modified by basic_manifest_handler +2025-05-12 23:35:13 - koi_net.processor.interface - INFO - Writing to cache: 'NEW', source: 'INTERNAL'> +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Change to node or edge, regenerating network graph +2025-05-12 23:35:13 - koi_net.network.graph - DEBUG - Generating network graph +2025-05-12 23:35:13 - koi_net.network.graph - DEBUG - Added node orn:koi-net.node:hackmd-sensor+c1311da2-023f-4ce5-a262-6b9a6db85dea +2025-05-12 23:35:13 - koi_net.network.graph - DEBUG - Done +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Calling network handler 'basic_network_output_filter' +2025-05-12 23:35:13 - koi_net.processor.default_handlers - DEBUG - Updating network targets with 'orn:koi-net.node' subscribers: [] +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Knowledge object modified by basic_network_output_filter +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - No network targets set +2025-05-12 23:35:13 - koi_net.processor.interface - DEBUG - Done +2025-05-12 23:35:13 - koi_net.core - DEBUG - Done +2025-05-12 23:35:13 - koi_net.core - DEBUG - I don't have any neighbors, reaching out to first contact http://127.0.0.1:8080/koi-net +2025-05-12 23:35:13 - koi_net.network.request_handler - DEBUG - Making request to http://127.0.0.1:8080/koi-net/events/broadcast +2025-05-12 23:35:13 - httpcore.connection - DEBUG - connect_tcp.started host='127.0.0.1' port=8080 local_address=None timeout=5.0 socket_options=None +2025-05-12 23:35:13 - httpcore.connection - DEBUG - connect_tcp.complete return_value= +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - send_request_headers.started request= +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - send_request_headers.complete +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - send_request_body.started request= +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - send_request_body.complete +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - receive_response_headers.started request= +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'date', b'Tue, 13 May 2025 04:35:13 GMT'), (b'server', b'uvicorn'), (b'content-length', b'4'), (b'content-type', b'application/json')]) +2025-05-12 23:35:13 - httpx - INFO - HTTP Request: POST http://127.0.0.1:8080/koi-net/events/broadcast "HTTP/1.1 200 OK" +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - receive_response_body.started request= +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - receive_response_body.complete +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - response_closed.started +2025-05-12 23:35:13 - httpcore.http11 - DEBUG - response_closed.complete +2025-05-12 23:35:13 - httpcore.connection - DEBUG - close.started +2025-05-12 23:35:13 - httpcore.connection - DEBUG - close.complete +2025-05-12 23:35:13 - koi_net.network.request_handler - INFO - Broadcasted 2 event(s) to 'http://127.0.0.1:8080/koi-net' +2025-05-12 23:35:13 - hackmd_sensor_node.server - INFO - KOI-net node orn:koi-net.node:hackmd-sensor+c1311da2-023f-4ce5-a262-6b9a6db85dea started. +2025-05-12 23:35:13 - hackmd_sensor_node.server - INFO - NETWORK SETUP: Beginning network initialization sequence... +2025-05-12 23:35:14 - hackmd_sensor_node.server - INFO - NETWORK SETUP: Initiating first contact with coordinator: http://127.0.0.1:8080/koi-net +2025-05-12 23:35:14 - koi_net.network.request_handler - DEBUG - Making request to http://127.0.0.1:8080/koi-net/bundles/fetch +2025-05-12 23:35:14 - httpcore.connection - DEBUG - connect_tcp.started host='127.0.0.1' port=8080 local_address=None timeout=5.0 socket_options=None +2025-05-12 23:35:14 - httpcore.connection - DEBUG - connect_tcp.complete return_value= +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - send_request_headers.started request= +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - send_request_headers.complete +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - send_request_body.started request= +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - send_request_body.complete +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - receive_response_headers.started request= +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'date', b'Tue, 13 May 2025 04:35:14 GMT'), (b'server', b'uvicorn'), (b'content-length', b'110'), (b'content-type', b'application/json')]) +2025-05-12 23:35:14 - httpx - INFO - HTTP Request: POST http://127.0.0.1:8080/koi-net/bundles/fetch "HTTP/1.1 200 OK" +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - receive_response_body.started request= +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - receive_response_body.complete +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - response_closed.started +2025-05-12 23:35:14 - httpcore.http11 - DEBUG - response_closed.complete +2025-05-12 23:35:14 - httpcore.connection - DEBUG - close.started +2025-05-12 23:35:14 - httpcore.connection - DEBUG - close.complete +2025-05-12 23:35:14 - koi_net.network.request_handler - INFO - Fetched 0 bundle(s) from 'http://127.0.0.1:8080/koi-net' +2025-05-12 23:35:14 - hackmd_sensor_node.server - WARNING - NETWORK SETUP: Could not fetch coordinator bundle +2025-05-12 23:35:14 - hackmd_sensor_node.server - INFO - NETWORK SETUP: Network initialization complete +2025-05-12 23:35:14 - hackmd_sensor_node.server - INFO - Performing initial HackMD backfill on startup... +2025-05-12 23:35:14 - hackmd_sensor_node.config - INFO - HackMD state file '.koi/hackmd/hackmd_state.json' not found. Starting empty. +2025-05-12 23:35:14 - hackmd_sensor_node.backfill - INFO - Starting HackMD backfill. +2025-05-12 23:35:14 - hackmd_sensor_node.backfill - INFO - Targeting specific HackMD notes for backfill: ['C1xso4C8SH-ZzDaloTq4Uw'] +2025-05-12 23:35:14 - hackmd_sensor_node.backfill - DEBUG - Fetching targeted note ID: C1xso4C8SH-ZzDaloTq4Uw +2025-05-12 23:35:14 - httpcore.connection - DEBUG - connect_tcp.started host='api.hackmd.io' port=443 local_address=None timeout=5.0 socket_options=None +2025-05-12 23:35:14 - httpcore.connection - DEBUG - connect_tcp.complete return_value= +2025-05-12 23:35:14 - httpcore.connection - DEBUG - start_tls.started ssl_context= server_hostname='api.hackmd.io' timeout=5.0 +2025-05-12 23:35:15 - httpcore.connection - DEBUG - start_tls.complete return_value= +2025-05-12 23:35:15 - httpcore.http11 - DEBUG - send_request_headers.started request= +2025-05-12 23:35:15 - httpcore.http11 - DEBUG - send_request_headers.complete +2025-05-12 23:35:15 - httpcore.http11 - DEBUG - send_request_body.started request= +2025-05-12 23:35:15 - httpcore.http11 - DEBUG - send_request_body.complete +2025-05-12 23:35:15 - httpcore.http11 - DEBUG - receive_response_headers.started request= +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Tue, 13 May 2025 04:35:15 GMT'), (b'Content-Type', b'application/json; charset=utf-8'), (b'Content-Length', b'19303'), (b'Connection', b'keep-alive'), (b'X-Powered-By', b'Express'), (b'Content-Language', b'dev'), (b'Set-Cookie', b'locale=dev; path=/; expires=Wed, 13 May 2026 04:35:15 GMT'), (b'X-HackMD-API-Version', b'1.0.0'), (b'X-RateLimit-UserLimit', b'2000'), (b'X-RateLimit-UserRemaining', b'1535'), (b'X-RateLimit-UserReset', b'1748735999999'), (b'X-Target-Scope', b'user'), (b'X-Target-Id', b'36c0c87e-77a1-4950-b6a5-b58b44164c3f'), (b'ETag', b'W/"4b67-ozqN1ldmz845iesIjYSqFboo/O8"')]) +2025-05-12 23:35:16 - httpx - INFO - HTTP Request: GET https://api.hackmd.io/v1/notes/C1xso4C8SH-ZzDaloTq4Uw "HTTP/1.1 200 OK" +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - receive_response_body.started request= +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - receive_response_body.complete +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - response_closed.started +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - response_closed.complete +2025-05-12 23:35:16 - httpcore.connection - DEBUG - close.started +2025-05-12 23:35:16 - httpcore.connection - DEBUG - close.complete +2025-05-12 23:35:16 - hackmd_sensor_node.backfill - INFO - Processing targeted note 'Working Concept — “Net-of-Nets” Model v0.4-r2' (ID: C1xso4C8SH-ZzDaloTq4Uw) - New or updated. +2025-05-12 23:35:16 - hackmd_sensor_node.backfill - DEBUG - Making backfill targeted note bundle orn:hackmd.note:C1xso4C8SH-ZzDaloTq4Uw available locally. +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Queued 'None', source: 'INTERNAL'> +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Dequeued 'None', source: 'INTERNAL'> +2025-05-12 23:35:16 - hackmd_sensor_node.backfill - INFO - HackMD backfill complete. Processed 1 notes, bundled 1 new/updated notes. +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Handling 'None', source: 'INTERNAL'> +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Calling rid handler 'basic_rid_handler' +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Calling manifest handler 'basic_manifest_handler' +2025-05-12 23:35:16 - koi_net.processor.default_handlers - DEBUG - RID previously unknown to me, labeling as 'NEW' +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Knowledge object modified by basic_manifest_handler +2025-05-12 23:35:16 - hackmd_sensor_node.config - DEBUG - Saved HackMD state to '.koi/hackmd/hackmd_state.json'. +2025-05-12 23:35:16 - koi_net.processor.interface - INFO - Writing to cache: 'NEW', source: 'INTERNAL'> +2025-05-12 23:35:16 - hackmd_sensor_node.server - INFO - Initial HackMD backfill completed. +2025-05-12 23:35:16 - hackmd_sensor_node.server - INFO - HackMD backfill loop starting. Interval: 600 seconds. +2025-05-12 23:35:16 - hackmd_sensor_node.config - INFO - Loaded HackMD state from '.koi/hackmd/hackmd_state.json': 1 notes tracked. +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Calling network handler 'basic_network_output_filter' +2025-05-12 23:35:16 - hackmd_sensor_node.server - INFO - Starting periodic HackMD backfill... +2025-05-12 23:35:16 - koi_net.processor.default_handlers - DEBUG - Updating network targets with 'orn:hackmd.note' subscribers: [] +2025-05-12 23:35:16 - hackmd_sensor_node.backfill - INFO - Starting HackMD backfill. +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Knowledge object modified by basic_network_output_filter +2025-05-12 23:35:16 - hackmd_sensor_node.backfill - INFO - Targeting specific HackMD notes for backfill: ['C1xso4C8SH-ZzDaloTq4Uw'] +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - No network targets set +2025-05-12 23:35:16 - hackmd_sensor_node.backfill - DEBUG - Fetching targeted note ID: C1xso4C8SH-ZzDaloTq4Uw +2025-05-12 23:35:16 - koi_net.processor.interface - DEBUG - Done +2025-05-12 23:35:16 - httpcore.connection - DEBUG - connect_tcp.started host='api.hackmd.io' port=443 local_address=None timeout=5.0 socket_options=None +2025-05-12 23:35:16 - httpcore.connection - DEBUG - connect_tcp.complete return_value= +2025-05-12 23:35:16 - httpcore.connection - DEBUG - start_tls.started ssl_context= server_hostname='api.hackmd.io' timeout=5.0 +2025-05-12 23:35:16 - httpcore.connection - DEBUG - start_tls.complete return_value= +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - send_request_headers.started request= +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - send_request_headers.complete +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - send_request_body.started request= +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - send_request_body.complete +2025-05-12 23:35:16 - httpcore.http11 - DEBUG - receive_response_headers.started request= +2025-05-12 23:35:17 - httpcore.http11 - DEBUG - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Tue, 13 May 2025 04:35:17 GMT'), (b'Content-Type', b'application/json; charset=utf-8'), (b'Content-Length', b'19303'), (b'Connection', b'keep-alive'), (b'X-Powered-By', b'Express'), (b'Content-Language', b'dev'), (b'Set-Cookie', b'locale=dev; path=/; expires=Wed, 13 May 2026 04:35:17 GMT'), (b'X-HackMD-API-Version', b'1.0.0'), (b'X-RateLimit-UserLimit', b'2000'), (b'X-RateLimit-UserRemaining', b'1534'), (b'X-RateLimit-UserReset', b'1748735999999'), (b'X-Target-Scope', b'user'), (b'X-Target-Id', b'36c0c87e-77a1-4950-b6a5-b58b44164c3f'), (b'ETag', b'W/"4b67-ozqN1ldmz845iesIjYSqFboo/O8"')]) +2025-05-12 23:35:17 - httpx - INFO - HTTP Request: GET https://api.hackmd.io/v1/notes/C1xso4C8SH-ZzDaloTq4Uw "HTTP/1.1 200 OK" +2025-05-12 23:35:17 - httpcore.http11 - DEBUG - receive_response_body.started request= +2025-05-12 23:35:17 - httpcore.http11 - DEBUG - receive_response_body.complete +2025-05-12 23:35:17 - httpcore.http11 - DEBUG - response_closed.started +2025-05-12 23:35:17 - httpcore.http11 - DEBUG - response_closed.complete +2025-05-12 23:35:17 - httpcore.connection - DEBUG - close.started +2025-05-12 23:35:17 - httpcore.connection - DEBUG - close.complete +2025-05-12 23:35:17 - hackmd_sensor_node.backfill - DEBUG - Skipping targeted note 'Working Concept — “Net-of-Nets” Model v0.4-r2' (ID: C1xso4C8SH-ZzDaloTq4Uw) - Already up-to-date. +2025-05-12 23:35:17 - hackmd_sensor_node.backfill - INFO - HackMD backfill complete. Processed 1 notes, bundled 0 new/updated notes. +2025-05-12 23:35:17 - hackmd_sensor_node.config - DEBUG - Saved HackMD state to '.koi/hackmd/hackmd_state.json'. +2025-05-12 23:35:17 - hackmd_sensor_node.server - INFO - Periodic HackMD backfill completed. +2025-05-12 23:35:17 - uvicorn.error - INFO - Application startup complete. +2025-05-12 23:35:17 - uvicorn.error - INFO - Uvicorn running on http://127.0.0.1:8002 (Press CTRL+C to quit) +2025-05-12 23:35:49 - uvicorn.error - INFO - Shutting down +2025-05-12 23:35:49 - uvicorn.error - INFO - Waiting for application shutdown. +2025-05-12 23:35:49 - hackmd_sensor_node.server - INFO - HackMD Sensor Node: FastAPI application shutdown... +2025-05-12 23:35:49 - koi_net.core - INFO - Stopping node... +2025-05-12 23:35:49 - koi_net.core - INFO - Waiting for kobj queue to empty (0 tasks remaining) +2025-05-12 23:35:49 - hackmd_sensor_node.server - INFO - KOI-net node orn:koi-net.node:hackmd-sensor+c1311da2-023f-4ce5-a262-6b9a6db85dea stopped. +2025-05-12 23:35:49 - hackmd_sensor_node.server - INFO - HackMD Sensor Node: Shutdown complete. +2025-05-12 23:35:49 - uvicorn.error - INFO - Application shutdown complete. +2025-05-12 23:35:49 - uvicorn.error - INFO - Finished server process [94462]