← back to Exo
changes include: ipc, general utilities, flakes stuff w/ just, autopull script
ef5c5b965417b6948889aeda8bb39abb37a29341 · 2025-08-25 17:33:40 +0100 · Andrei Cravtov
Files touched
M .flake-modules/go-forwarder.nixA .flake-modules/just-flake.nixM .gitignoreM .idea/exo-v2.imlM flake.lockM flake.nixM justfileM networking/forwarder/go.modM networking/forwarder/go.sumA networking/forwarder/lib/go.modA networking/forwarder/lib/go.sumA networking/forwarder/lib/ipc/flock_mutex.goA networking/forwarder/lib/ipc/flock_mutex_test.goA networking/forwarder/lib/ipc/pipe_duplex.goA networking/forwarder/lib/ipc/pipe_duplex_test.goA networking/forwarder/lib/libp2pext/dm/config.goA networking/forwarder/lib/libp2pext/dm/dm.goA networking/forwarder/lib/libp2pext/dm/dm_test.goA networking/forwarder/lib/libp2pext/dm/internal.goA networking/forwarder/lib/util.goM pyproject.tomlM scripts/pyproject.tomlM scripts/src/exo_scripts/read_events.pyA scripts/watch-pull-restart.pyM src/exo/engines/mlx/auto_parallel.pyM src/exo/engines/mlx/utils_mlx.pyM src/exo/master/forwarder_supervisor.pyM src/exo/master/main.pyM src/exo/master/tests/test_forwarder_supervisor.pyM src/exo/master/tests/test_master.pyM src/exo/shared/constants.pyM src/exo/shared/db/sqlite/event_log_manager.pyA src/exo/shared/ipc/__init__.pyA src/exo/shared/ipc/file_mutex/__init__.pyA src/exo/shared/ipc/file_mutex/flock_mutex.pyA src/exo/shared/ipc/pipe_duplex.pyR097 src/exo/shared/utils.py src/exo/shared/keypair.pyA src/exo/shared/tests/test_flock_mutex.pyM src/exo/shared/tests/test_node_id_persistence.pyM src/exo/shared/types/events/_events.pyA src/exo/shared/utils/__init__.pyA src/exo/shared/utils/fs.pyA src/exo/shared/utils/phantom.pyA src/exo/shared/utils/pydantic_ext.pyA src/exo/shared/utils/reactive.pyM src/exo/worker/main.pyM src/exo/worker/runner/runner.pyM src/exo/worker/tests/test_spinup_timeout.pyM uv.lockD worker/pyproject.toml
Diff
commit ef5c5b965417b6948889aeda8bb39abb37a29341
Author: Andrei Cravtov <the.andrei.cravtov@gmail.com>
Date: Mon Aug 25 17:33:40 2025 +0100
changes include: ipc, general utilities, flakes stuff w/ just, autopull script
---
.flake-modules/go-forwarder.nix | 42 +-
.flake-modules/just-flake.nix | 54 +++
.gitignore | 8 +-
.idea/exo-v2.iml | 9 +-
flake.lock | 16 +
flake.nix | 5 +
justfile | 8 +
networking/forwarder/go.mod | 19 +-
networking/forwarder/go.sum | 38 +-
networking/forwarder/lib/go.mod | 106 ++++++
networking/forwarder/lib/go.sum | 443 ++++++++++++++++++++++
networking/forwarder/lib/ipc/flock_mutex.go | 208 ++++++++++
networking/forwarder/lib/ipc/flock_mutex_test.go | 86 +++++
networking/forwarder/lib/ipc/pipe_duplex.go | 400 +++++++++++++++++++
networking/forwarder/lib/ipc/pipe_duplex_test.go | 85 +++++
networking/forwarder/lib/libp2pext/dm/config.go | 38 ++
networking/forwarder/lib/libp2pext/dm/dm.go | 57 +++
networking/forwarder/lib/libp2pext/dm/dm_test.go | 88 +++++
networking/forwarder/lib/libp2pext/dm/internal.go | 151 ++++++++
networking/forwarder/lib/util.go | 52 +++
pyproject.toml | 1 +
scripts/pyproject.toml | 5 +-
scripts/src/exo_scripts/read_events.py | 90 +++--
scripts/watch-pull-restart.py | 284 ++++++++++++++
src/exo/engines/mlx/auto_parallel.py | 2 +-
src/exo/engines/mlx/utils_mlx.py | 10 +-
src/exo/master/forwarder_supervisor.py | 1 +
src/exo/master/main.py | 6 +-
src/exo/master/tests/test_forwarder_supervisor.py | 4 +-
src/exo/master/tests/test_master.py | 10 +-
src/exo/shared/constants.py | 2 +
src/exo/shared/db/sqlite/event_log_manager.py | 3 +-
src/exo/shared/ipc/__init__.py | 14 +
src/exo/shared/ipc/file_mutex/__init__.py | 4 +
src/exo/shared/ipc/file_mutex/flock_mutex.py | 147 +++++++
src/exo/shared/ipc/pipe_duplex.py | 415 ++++++++++++++++++++
src/exo/shared/{utils.py => keypair.py} | 8 +-
src/exo/shared/tests/test_flock_mutex.py | 48 +++
src/exo/shared/tests/test_node_id_persistence.py | 2 +-
src/exo/shared/types/events/_events.py | 4 +-
src/exo/shared/utils/__init__.py | 18 +
src/exo/shared/utils/fs.py | 34 ++
src/exo/shared/utils/phantom.py | 14 +
src/exo/shared/utils/pydantic_ext.py | 52 +++
src/exo/shared/utils/reactive.py | 32 ++
src/exo/worker/main.py | 2 +-
src/exo/worker/runner/runner.py | 2 +-
src/exo/worker/tests/test_spinup_timeout.py | 4 +-
uv.lock | 56 +--
worker/pyproject.toml | 18 -
50 files changed, 3024 insertions(+), 181 deletions(-)
diff --git a/.flake-modules/go-forwarder.nix b/.flake-modules/go-forwarder.nix
index 647e54ee..34a38cf1 100644
--- a/.flake-modules/go-forwarder.nix
+++ b/.flake-modules/go-forwarder.nix
@@ -33,37 +33,39 @@
...
}:
let
- flakeRoot = nixpkgs-lib.getExe config.flake-root.package;
-
- # Build the networking/forwarder Go utility.
- forwarder = pkgs.buildGoModule {
- pname = "exo-forwarder";
- version = "0.1.0";
- src = "${flakeRoot}/networking/forwarder";
-
- vendorHash = "sha256-BXIGg2QYqHDz2TNe8hLAGC6jVlffp9766H+WdkkuVgA=";
-
- # Only the main package at the repository root needs building.
- subPackages = [ "." ];
- };
+# flakeRoot = nixpkgs-lib.getExe config.flake-root.package;
+#
+# # Build the networking/forwarder Go utility.
+# forwarder = pkgs.buildGoModule {
+# pname = "exo-forwarder";
+# version = "0.1.0";
+# src = "${flakeRoot}/networking/forwarder";
+#
+# vendorHash = "sha256-BXIGg2QYqHDz2TNe8hLAGC6jVlffp9766H+WdkkuVgA=";
+#
+# # Only the main package at the repository root needs building.
+# subPackages = [ "." ];
+# };
in
{
packages = {
- inherit forwarder;
+# inherit forwarder;
};
apps = {
- forwarder = {
- type = "app";
- program = "${forwarder}/bin/forwarder";
- };
+# forwarder = {
+# type = "app";
+# program = "${forwarder}/bin/forwarder";
+# };
};
make-shells.default = {
# Go 1.24 compiler – align with go.mod
packages = [ pkgs.go_1_24 ];
- shellHook = "export GOPATH=$FLAKE_ROOT/.go_cache";
+ shellHook = ''
+ GOPATH="''$(${nixpkgs-lib.getExe config.flake-root.package})"/.go_cache
+ export GOPATH
+ '';
};
};
}
-
diff --git a/.flake-modules/just-flake.nix b/.flake-modules/just-flake.nix
new file mode 100644
index 00000000..2208a58c
--- /dev/null
+++ b/.flake-modules/just-flake.nix
@@ -0,0 +1,54 @@
+# Provides pretty banner & command index for this flake
+
+# Top-level parameters that are bound to the provider flake
+# These are passed from `flake.nix` using importApply
+{
+ localSelf,
+ flake-parts-lib,
+ nixpkgs-lib,
+ just-flake,
+ ...
+}:
+
+# These values would bind to the consumer flake when this flake module is imported:
+{
+ config,
+ self,
+ inputs,
+ getSystem,
+ moduleWithSystem,
+ withSystem,
+ ...
+}:
+
+# The actual flake-parts module configuration
+{
+ imports = [ just-flake.flakeModule ];
+ perSystem =
+ {
+ config,
+ self',
+ inputs',
+ pkgs,
+ system,
+ ...
+ }:
+ {
+ just-flake.features = {
+ # treefmt.enable = true;
+ # rust.enable = true;
+ # convco.enable = true;
+ # hello = {
+ # enable = true;
+ # justfile = ''
+ # hello:
+ # echo Hello World
+ # '';
+ # };
+ };
+
+ make-shells.default = {
+ inputsFrom = [ config.just-flake.outputs.devShell ];
+ };
+ };
+}
diff --git a/.gitignore b/.gitignore
index 3e73b059..200f8908 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,15 +12,15 @@ hosts_*.json
# TODO figure out how to properly solve the issue with these target directories showing up
networking/target/
networking/topology/target/
+rust/target/
+rust/Cargo.lock
build/
+dist/
*.xcuserstate
-rust/target/
-rust/Cargo.lock
-
.DS_Store
*/.DS_Store
# Says this symlink should be git-ignored https://github.com/juspay/just-flake
-just-flake.just
+just-flake.just
\ No newline at end of file
diff --git a/.idea/exo-v2.iml b/.idea/exo-v2.iml
index e4d93c64..5357eaa9 100644
--- a/.idea/exo-v2.iml
+++ b/.idea/exo-v2.iml
@@ -5,15 +5,16 @@
<configuration sdkName="Python 3.13 virtualenv at ~/Desktop/exo/.venv" />
</facet>
</component>
+ <component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
- <sourceFolder url="file://$MODULE_DIR$/engines/mlx" isTestSource="false" />
- <sourceFolder url="file://$MODULE_DIR$/master" isTestSource="false" />
- <sourceFolder url="file://$MODULE_DIR$/shared" isTestSource="false" />
- <sourceFolder url="file://$MODULE_DIR$/worker" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/scripts/src" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/.venv" />
<excludeFolder url="file://$MODULE_DIR$/.direnv" />
<excludeFolder url="file://$MODULE_DIR$/build" />
+ <excludeFolder url="file://$MODULE_DIR$/dist" />
+ <excludeFolder url="file://$MODULE_DIR$/.go_cache" />
</content>
<orderEntry type="jdk" jdkName="Python 3.13 (exo)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
diff --git a/flake.lock b/flake.lock
index 35e1853d..bc30d2b3 100644
--- a/flake.lock
+++ b/flake.lock
@@ -51,6 +51,21 @@
"type": "github"
}
},
+ "just-flake": {
+ "locked": {
+ "lastModified": 1713316411,
+ "narHash": "sha256-NkJfU6H+6vgHkPtZ2ESbZ/h2wnsDQrZvB4vbdUIBx8Q=",
+ "owner": "juspay",
+ "repo": "just-flake",
+ "rev": "0e33952a4bcd16cd54ee3aba8111606c237d4526",
+ "type": "github"
+ },
+ "original": {
+ "owner": "juspay",
+ "repo": "just-flake",
+ "type": "github"
+ }
+ },
"make-shell": {
"inputs": {
"flake-compat": "flake-compat"
@@ -89,6 +104,7 @@
"inputs": {
"flake-parts": "flake-parts",
"flake-root": "flake-root",
+ "just-flake": "just-flake",
"make-shell": "make-shell",
"nixpkgs": "nixpkgs"
}
diff --git a/flake.nix b/flake.nix
index 17253618..0098a869 100644
--- a/flake.nix
+++ b/flake.nix
@@ -17,6 +17,9 @@
# 1. ${lib.getExe config.flake-root.package}
# 2. $FLAKE_ROOT environment-varible
flake-root.url = "github:srid/flake-root";
+
+ # Provides flake integration with [Just](https://just.systems/man/en/)
+ just-flake.url = "github:juspay/just-flake";
};
outputs =
@@ -47,6 +50,7 @@
# instantiate all the flake modules, passing custom arguments to them as needed
flakeModules = {
flakeRoot = importApply' ./.flake-modules/flake-root.nix { inherit (inputs) flake-root; };
+ justFlake = importApply' ./.flake-modules/just-flake.nix { inherit (inputs) just-flake; };
goForwarder = importApply' ./.flake-modules/go-forwarder.nix { };
};
in
@@ -54,6 +58,7 @@
imports = [
inputs.make-shell.flakeModules.default
flakeModules.flakeRoot
+ flakeModules.justFlake
flakeModules.goForwarder
./.flake-modules/macmon.nix
];
diff --git a/justfile b/justfile
index 53aaf70c..1b84e2eb 100644
--- a/justfile
+++ b/justfile
@@ -1,3 +1,5 @@
+import 'just-flake.just'
+
default:
@just --list
@@ -45,3 +47,9 @@ run n="1" clean="false":
if [ "{{clean}}" = "true" ]; then ./run.sh -rc; else ./run.sh -r; fi; \
done; \
fi
+
+# remote debugging auto-runner command: TODO: find better place to put this??
+# -> this pulls from upstream and wipes .exo folder, rebuilds & restarts
+# -> TODO: maybe add a sync step for python deps ??
+autorun-master:
+ uv run scripts/watch-pull-restart.py --cmd "uv run exo-master" --restart-cmd "rm -rf ~/.exo && just build-forwarder"
\ No newline at end of file
diff --git a/networking/forwarder/go.mod b/networking/forwarder/go.mod
index 47079c0f..51064579 100644
--- a/networking/forwarder/go.mod
+++ b/networking/forwarder/go.mod
@@ -1,12 +1,14 @@
module forwarder
-go 1.24.3
+go 1.24.5
+
+replace lib => ./lib
replace forwarder/src => ./src
require (
github.com/google/uuid v1.6.0
- github.com/libp2p/go-libp2p v0.42.1
+ github.com/libp2p/go-libp2p v0.43.0
github.com/libp2p/go-libp2p-pubsub v0.14.2
github.com/mattn/go-sqlite3 v1.14.28
github.com/multiformats/go-multiaddr v0.16.0
@@ -22,10 +24,8 @@ require (
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
github.com/flynn/noise v1.1.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
- github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/gopacket v1.1.19 // indirect
- github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/huin/goupnp v1.3.0 // indirect
@@ -61,7 +61,6 @@ require (
github.com/multiformats/go-multistream v0.6.1 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
- github.com/onsi/ginkgo/v2 v2.23.4 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/pion/datachannel v1.5.10 // indirect
github.com/pion/dtls/v2 v2.2.12 // indirect
@@ -88,11 +87,11 @@ require (
github.com/prometheus/common v0.64.0 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
- github.com/quic-go/quic-go v0.52.0 // indirect
- github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect
+ github.com/quic-go/quic-go v0.54.0 // indirect
+ github.com/quic-go/webtransport-go v0.9.0 // indirect
+ github.com/rogpeppe/go-internal v1.13.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/wlynxg/anet v0.0.5 // indirect
- go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/dig v1.19.0 // indirect
go.uber.org/fx v1.24.0 // indirect
go.uber.org/mock v0.5.2 // indirect
@@ -102,8 +101,8 @@ require (
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
- golang.org/x/sync v0.15.0 // indirect
- golang.org/x/sys v0.33.0 // indirect
+ golang.org/x/sync v0.16.0 // indirect
+ golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/time v0.12.0 // indirect
golang.org/x/tools v0.34.0 // indirect
diff --git a/networking/forwarder/go.sum b/networking/forwarder/go.sum
index 5ba5ce9e..2d13eb91 100644
--- a/networking/forwarder/go.sum
+++ b/networking/forwarder/go.sum
@@ -39,10 +39,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
-github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -62,8 +58,6 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a h1://KbezygeMJZCSHH+HgUZiTeSoiuFspbMg1ge+eFj18=
-github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
@@ -109,8 +103,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw=
github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc=
-github.com/libp2p/go-libp2p v0.42.1 h1:Rt8+5thie729NQk1gx1h/2t/+VIafWcqR1I+Kvw+UTg=
-github.com/libp2p/go-libp2p v0.42.1/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro=
+github.com/libp2p/go-libp2p v0.43.0 h1:b2bg2cRNmY4HpLK8VHYQXLX2d3iND95OjodLFymvqXU=
+github.com/libp2p/go-libp2p v0.43.0/go.mod h1:IiSqAXDyP2sWH+J2gs43pNmB/y4FOi2XQPbsb+8qvzc=
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
github.com/libp2p/go-libp2p-pubsub v0.14.2 h1:nT5lFHPQOFJcp9CW8hpKtvbpQNdl2udJuzLQWbgRum8=
@@ -181,10 +175,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
-github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
-github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU=
-github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
@@ -233,8 +223,6 @@ github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZ
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
-github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
@@ -249,12 +237,12 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.52.0 h1:/SlHrCRElyaU6MaEPKqKr9z83sBg2v4FLLvWM+Z47pA=
-github.com/quic-go/quic-go v0.52.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ=
-github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg=
-github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg=
+github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
+github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70=
+github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
@@ -303,8 +291,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
-go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
-go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
@@ -385,8 +371,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
-golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -408,8 +394,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
diff --git a/networking/forwarder/lib/go.mod b/networking/forwarder/lib/go.mod
new file mode 100644
index 00000000..9f10985e
--- /dev/null
+++ b/networking/forwarder/lib/go.mod
@@ -0,0 +1,106 @@
+module lib
+
+go 1.24.5
+
+require (
+ github.com/ipfs/go-log/v2 v2.6.0
+ github.com/stretchr/testify v1.10.0
+ golang.org/x/sys v0.35.0
+)
+
+require (
+ github.com/benbjohnson/clock v1.3.5 // indirect
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
+ github.com/flynn/noise v1.1.0 // indirect
+ github.com/francoispqt/gojay v1.2.13 // indirect
+ github.com/google/gopacket v1.1.19 // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/gorilla/websocket v1.5.3 // indirect
+ github.com/huin/goupnp v1.3.0 // indirect
+ github.com/ipfs/go-cid v0.5.0 // indirect
+ github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+ github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+ github.com/klauspost/compress v1.18.0 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.10 // indirect
+ github.com/koron/go-ssdp v0.0.6 // indirect
+ github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+ github.com/libp2p/go-flow-metrics v0.2.0 // indirect
+ github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
+ github.com/libp2p/go-msgio v0.3.0 // indirect
+ github.com/libp2p/go-netroute v0.2.2 // indirect
+ github.com/libp2p/go-reuseport v0.4.0 // indirect
+ github.com/libp2p/go-yamux/v5 v5.0.1 // indirect
+ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/miekg/dns v1.1.66 // indirect
+ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+ github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
+ github.com/minio/sha256-simd v1.0.1 // indirect
+ github.com/mr-tron/base58 v1.2.0 // indirect
+ github.com/multiformats/go-base32 v0.1.0 // indirect
+ github.com/multiformats/go-base36 v0.2.0 // indirect
+ github.com/multiformats/go-multiaddr v0.16.0 // indirect
+ github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect
+ github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+ github.com/multiformats/go-multibase v0.2.0 // indirect
+ github.com/multiformats/go-multicodec v0.9.1 // indirect
+ github.com/multiformats/go-multihash v0.2.3 // indirect
+ github.com/multiformats/go-multistream v0.6.1 // indirect
+ github.com/multiformats/go-varint v0.0.7 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+ github.com/pion/datachannel v1.5.10 // indirect
+ github.com/pion/dtls/v2 v2.2.12 // indirect
+ github.com/pion/dtls/v3 v3.0.6 // indirect
+ github.com/pion/ice/v4 v4.0.10 // indirect
+ github.com/pion/interceptor v0.1.40 // indirect
+ github.com/pion/logging v0.2.3 // indirect
+ github.com/pion/mdns/v2 v2.0.7 // indirect
+ github.com/pion/randutil v0.1.0 // indirect
+ github.com/pion/rtcp v1.2.15 // indirect
+ github.com/pion/rtp v1.8.19 // indirect
+ github.com/pion/sctp v1.8.39 // indirect
+ github.com/pion/sdp/v3 v3.0.13 // indirect
+ github.com/pion/srtp/v3 v3.0.6 // indirect
+ github.com/pion/stun v0.6.1 // indirect
+ github.com/pion/stun/v3 v3.0.0 // indirect
+ github.com/pion/transport/v2 v2.2.10 // indirect
+ github.com/pion/transport/v3 v3.0.7 // indirect
+ github.com/pion/turn/v4 v4.0.2 // indirect
+ github.com/pion/webrtc/v4 v4.1.2 // indirect
+ github.com/prometheus/client_golang v1.22.0 // indirect
+ github.com/prometheus/client_model v0.6.2 // indirect
+ github.com/prometheus/common v0.64.0 // indirect
+ github.com/prometheus/procfs v0.16.1 // indirect
+ github.com/quic-go/qpack v0.5.1 // indirect
+ github.com/quic-go/quic-go v0.54.0 // indirect
+ github.com/quic-go/webtransport-go v0.9.0 // indirect
+ github.com/spaolacci/murmur3 v1.1.0 // indirect
+ github.com/wlynxg/anet v0.0.5 // indirect
+ go.uber.org/dig v1.19.0 // indirect
+ go.uber.org/fx v1.24.0 // indirect
+ go.uber.org/mock v0.5.2 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
+ go.uber.org/zap v1.27.0 // indirect
+ golang.org/x/crypto v0.39.0 // indirect
+ golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
+ golang.org/x/mod v0.25.0 // indirect
+ golang.org/x/net v0.41.0 // indirect
+ golang.org/x/text v0.26.0 // indirect
+ golang.org/x/time v0.12.0 // indirect
+ golang.org/x/tools v0.34.0 // indirect
+ google.golang.org/protobuf v1.36.6 // indirect
+ lukechampine.com/blake3 v1.4.1 // indirect
+)
+
+require (
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/libp2p/go-libp2p v0.43.0
+ github.com/pdgendt/cobs v1.1.0
+ github.com/pmezard/go-difflib v1.0.0 // indirect
+ golang.org/x/sync v0.16.0
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+)
diff --git a/networking/forwarder/lib/go.sum b/networking/forwarder/lib/go.sum
new file mode 100644
index 00000000..b4e5ba17
--- /dev/null
+++ b/networking/forwarder/lib/go.sum
@@ -0,0 +1,443 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
+dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
+dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
+dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
+dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
+github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
+github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU=
+github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U=
+github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
+github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
+github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
+github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
+github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
+github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
+github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=
+github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk=
+github.com/ipfs/go-log/v2 v2.6.0 h1:2Nu1KKQQ2ayonKp4MPo6pXCjqw1ULc9iohRqWV5EYqg=
+github.com/ipfs/go-log/v2 v2.6.0/go.mod h1:p+Efr3qaY5YXpx9TX7MoLCSEZX5boSWj9wh86P5HJa8=
+github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
+github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk=
+github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk=
+github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
+github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU=
+github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=
+github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
+github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw=
+github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc=
+github.com/libp2p/go-libp2p v0.43.0 h1:b2bg2cRNmY4HpLK8VHYQXLX2d3iND95OjodLFymvqXU=
+github.com/libp2p/go-libp2p v0.43.0/go.mod h1:IiSqAXDyP2sWH+J2gs43pNmB/y4FOi2XQPbsb+8qvzc=
+github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
+github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
+github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA=
+github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg=
+github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0=
+github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM=
+github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFPuZ8=
+github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE=
+github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s=
+github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU=
+github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg=
+github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU=
+github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk=
+github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
+github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE=
+github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE=
+github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8=
+github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms=
+github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc=
+github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU=
+github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc=
+github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
+github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=
+github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI=
+github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
+github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
+github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
+github.com/multiformats/go-multiaddr v0.16.0 h1:oGWEVKioVQcdIOBlYM8BH1rZDWOGJSqr9/BKl6zQ4qc=
+github.com/multiformats/go-multiaddr v0.16.0/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0=
+github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M=
+github.com/multiformats/go-multiaddr-dns v0.4.1/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc=
+github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=
+github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo=
+github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
+github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
+github.com/multiformats/go-multicodec v0.9.1 h1:x/Fuxr7ZuR4jJV4Os5g444F7xC4XmyUaT/FWtE+9Zjo=
+github.com/multiformats/go-multicodec v0.9.1/go.mod h1:LLWNMtyV5ithSBUo3vFIMaeDy+h3EbkMTek1m+Fybbo=
+github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
+github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
+github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
+github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ=
+github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw=
+github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
+github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
+github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
+github.com/pdgendt/cobs v1.1.0 h1:gGeI8VUIMCz5jAWoEi24UZv+vsQwiOSjoJuRY4jKnxg=
+github.com/pdgendt/cobs v1.1.0/go.mod h1:AdxrOLm724a1y0E1RQn6+PtMjLUXgBM4FQJ9lm+/h3E=
+github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o=
+github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M=
+github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
+github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
+github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
+github.com/pion/dtls/v3 v3.0.6 h1:7Hkd8WhAJNbRgq9RgdNh1aaWlZlGpYTzdqjy9x9sK2E=
+github.com/pion/dtls/v3 v3.0.6/go.mod h1:iJxNQ3Uhn1NZWOMWlLxEEHAN5yX7GyPvvKw04v9bzYU=
+github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4=
+github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
+github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4=
+github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic=
+github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
+github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI=
+github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90=
+github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
+github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
+github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
+github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
+github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo=
+github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0=
+github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c=
+github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk=
+github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE=
+github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE=
+github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4=
+github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E=
+github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4=
+github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY=
+github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4=
+github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8=
+github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
+github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU=
+github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
+github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
+github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q=
+github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
+github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
+github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
+github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps=
+github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs=
+github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54=
+github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4=
+github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
+github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
+github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg=
+github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
+github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70=
+github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
+github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
+github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
+github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
+github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
+github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
+github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
+github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
+github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
+github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
+github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
+github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
+github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
+github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
+github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
+github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
+github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
+github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
+github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
+github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
+github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
+github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
+github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
+github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
+go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
+go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
+go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
+go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
+golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
+golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
+golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
+golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
+golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
+golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg=
+lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo=
+sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
diff --git a/networking/forwarder/lib/ipc/flock_mutex.go b/networking/forwarder/lib/ipc/flock_mutex.go
new file mode 100644
index 00000000..a15775ff
--- /dev/null
+++ b/networking/forwarder/lib/ipc/flock_mutex.go
@@ -0,0 +1,208 @@
+//go:build unix
+
+package ipc
+
+import (
+ "errors"
+ "syscall"
+ "time"
+
+ "golang.org/x/sys/unix"
+)
+
+var (
+ ErrFileDescriptorAlreadyOpen = errors.New("file descriptor not open")
+ ErrFileDescriptorNotOpen = errors.New("file descriptor not open")
+ ErrLockAlreadyHeld = errors.New("lock already held")
+ ErrLockNotHeld = errors.New("lock not held")
+)
+
+const (
+ // open in read-write mode, creates file if it doesn't exist already,
+ // closes this file descriptor in any children processes (prevents FD leaking),
+ // truncates this file on opening (lock-files shouldn't hold content FOR NOW!!!)
+ //
+ // SEE: https://man7.org/linux/man-pages/man2/openat.2.html
+ flockMutexOpenFlags int = syscall.O_RDWR | syscall.O_CREAT | syscall.O_CLOEXEC | syscall.O_TRUNC
+
+ // 0x644 mode flags -> user has read-write permissions, others have read permission only
+ // SEE: https://man7.org/linux/man-pages/man2/openat.2.html
+ flockMutexModeFlags uint32 = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP | syscall.S_IROTH
+
+ // default poll-interval for spin-blocking lock
+ flockMutexPollInterval = 50 * time.Millisecond
+)
+
+type LockType int
+
+const (
+ ReadLock LockType = syscall.LOCK_SH
+ WriteLock LockType = syscall.LOCK_EX
+ LockMissing LockType = -1
+)
+
+type AcquireMode int
+
+const (
+ OsBlocking AcquireMode = iota
+ SpinBlocking
+ NonBlocking
+)
+
+type FlockMutex struct {
+ filePath string
+ fd int
+ lockHeld LockType
+}
+
+func NewFlockMutex(filePath string) *FlockMutex {
+ return &FlockMutex{
+ filePath: filePath,
+ fd: -1,
+ lockHeld: LockMissing,
+ }
+}
+
+func (mu *FlockMutex) openFd() error {
+ if mu.fd != -1 {
+ return ErrFileDescriptorAlreadyOpen
+ }
+ // TODO: ensure_directory_exists(mu.filePath)
+
+ // open file & TRY to change permissions to `modeFlags` flags
+ fd, err := unix.Open(mu.filePath, flockMutexOpenFlags, flockMutexModeFlags)
+ if err != nil {
+ return err
+ } else {
+ mu.fd = fd
+ _ = unix.Fchmod(fd, flockMutexModeFlags) // This locked is not owned by this UID
+ }
+ return nil
+}
+
+func (mu *FlockMutex) closeFd() error {
+ if mu.fd == -1 {
+ return ErrFileDescriptorNotOpen
+ }
+
+ if err := unix.Close(mu.fd); err != nil {
+ mu.fd = -1
+ return err
+ }
+
+ mu.fd = -1
+ return nil
+}
+
+func (mu *FlockMutex) acquire(lockType LockType, blocking bool) (bool, error) {
+ // enforce preconditions/sanity checks
+ if mu.fd == -1 {
+ return false, ErrFileDescriptorNotOpen
+ }
+ if mu.lockHeld != LockMissing {
+ return false, ErrLockAlreadyHeld
+ }
+
+ // create flags for acquiring lock
+ var flags = int(lockType)
+ if !blocking {
+ flags |= syscall.LOCK_NB
+ }
+
+ // continually try to acquire lock (since it may fail due to interrupts)
+ for {
+ if err := unix.Flock(mu.fd, flags); err != nil {
+ if errno, ok := err.(unix.Errno); ok {
+ // call interrupted by signal -> try again
+ if errno == unix.EINTR {
+ continue
+ }
+
+ // file is locked & non-blocking is enabled -> return false to indicate
+ if errno == unix.EWOULDBLOCK {
+ return false, nil
+ }
+ }
+
+ // unhandleable errors -> close FD & return error
+ _ = mu.closeFd() // TODO: how to merge Go errors ???
+ return false, err
+ }
+ break
+ }
+
+ // set lock-type held
+ mu.lockHeld = lockType
+ return true, nil
+}
+
+func (mu *FlockMutex) release() error {
+ // enforce preconditions/sanity checks
+ if mu.fd == -1 {
+ return ErrFileDescriptorNotOpen
+ }
+ if mu.lockHeld == LockMissing {
+ return ErrLockNotHeld
+ }
+
+ // continually try to release lock (since it may fail due to interrupts)
+ for {
+ if err := unix.Flock(mu.fd, syscall.LOCK_UN); err != nil {
+ if errno, ok := err.(unix.Errno); ok {
+ // call interrupted by signal -> try again
+ if errno == unix.EINTR {
+ continue
+ }
+ }
+
+ // unhandleable errors -> close FD & return error
+ mu.lockHeld = LockMissing
+ _ = mu.closeFd() // TODO: how to merge Go errors ???
+ return err
+ }
+ break
+ }
+
+ mu.lockHeld = LockMissing
+ return nil
+}
+
+func (mu *FlockMutex) Acquire(lockType LockType, acquireMode AcquireMode) (bool, error) {
+ // open file if missing
+ if mu.fd == -1 {
+ if err := mu.openFd(); err != nil {
+ return false, err
+ }
+ }
+
+ // OS-blocking & non-blocking is direct passthrough to private function
+ switch acquireMode {
+ case OsBlocking:
+ return mu.acquire(lockType, true)
+ case NonBlocking:
+ return mu.acquire(lockType, false)
+ }
+
+ // spin-blocking works by trying to acquire the lock in non-blocking mode, and retrying until success
+ for {
+ locked, err := mu.acquire(lockType, false)
+ if err != nil {
+ return false, err
+ }
+ if locked {
+ return true, err
+ }
+ time.Sleep(flockMutexPollInterval)
+ }
+}
+
+func (mu *FlockMutex) Release(lockType LockType, acquireMode AcquireMode) error {
+ if err := mu.release(); err != nil {
+ _ = mu.closeFd() // TODO: how to merge Go errors ???
+ return err
+ }
+ if err := mu.closeFd(); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/networking/forwarder/lib/ipc/flock_mutex_test.go b/networking/forwarder/lib/ipc/flock_mutex_test.go
new file mode 100644
index 00000000..b0cb136f
--- /dev/null
+++ b/networking/forwarder/lib/ipc/flock_mutex_test.go
@@ -0,0 +1,86 @@
+//go:build unix
+
+package ipc
+
+import (
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func check(t *testing.T, err error) {
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+}
+
+func makeTempPath(t *testing.T, pattern string) string {
+ f, err := os.CreateTemp("", pattern)
+ check(t, err)
+ name := f.Name()
+ defer os.Remove(name)
+ return name
+}
+
+func TestLockHeld(t *testing.T) {
+ path := makeTempPath(t, "testing_flock.lock")
+ defer os.Remove(path)
+ mu := NewFlockMutex(path)
+
+ assert.Equal(t, LockMissing, mu.lockHeld)
+
+ acquired, err := mu.Acquire(WriteLock, SpinBlocking)
+ check(t, err)
+ assert.True(t, acquired)
+ assert.Equal(t, WriteLock, mu.lockHeld)
+ check(t, mu.release())
+
+ assert.Equal(t, LockMissing, mu.lockHeld)
+
+ acquired, err = mu.Acquire(ReadLock, SpinBlocking)
+ check(t, err)
+ assert.True(t, acquired)
+ assert.Equal(t, ReadLock, mu.lockHeld)
+ check(t, mu.release())
+
+ assert.Equal(t, LockMissing, mu.lockHeld)
+}
+
+func TestNoReentrantLock(t *testing.T) {
+ path := makeTempPath(t, "testing_flock.lock")
+ defer os.Remove(path)
+ mu := NewFlockMutex(path)
+
+ // no write-lock reentrancy
+ acquired, err := mu.Acquire(WriteLock, SpinBlocking)
+ check(t, err)
+ assert.True(t, acquired)
+ {
+ acquired, err = mu.Acquire(WriteLock, SpinBlocking)
+ assert.False(t, acquired)
+ assert.Equal(t, ErrLockAlreadyHeld, err)
+ }
+ {
+ acquired, err = mu.Acquire(ReadLock, SpinBlocking)
+ assert.False(t, acquired)
+ assert.Equal(t, ErrLockAlreadyHeld, err)
+ }
+ check(t, mu.release())
+
+ // no read-lock reentrancy
+ acquired, err = mu.Acquire(ReadLock, SpinBlocking)
+ check(t, err)
+ assert.True(t, acquired)
+ {
+ acquired, err = mu.Acquire(WriteLock, SpinBlocking)
+ assert.False(t, acquired)
+ assert.Equal(t, ErrLockAlreadyHeld, err)
+ }
+ {
+ acquired, err = mu.Acquire(ReadLock, SpinBlocking)
+ assert.False(t, acquired)
+ assert.Equal(t, ErrLockAlreadyHeld, err)
+ }
+ check(t, mu.release())
+}
diff --git a/networking/forwarder/lib/ipc/pipe_duplex.go b/networking/forwarder/lib/ipc/pipe_duplex.go
new file mode 100644
index 00000000..eeb0a396
--- /dev/null
+++ b/networking/forwarder/lib/ipc/pipe_duplex.go
@@ -0,0 +1,400 @@
+//go:build unix
+
+package ipc
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io/fs"
+ "lib"
+ "log"
+ "os"
+ "sync"
+ "syscall"
+ "time"
+
+ "github.com/pdgendt/cobs"
+ "golang.org/x/sync/errgroup"
+ "golang.org/x/sys/unix"
+)
+
+var (
+ ErrInOutPipesAreSame = errors.New("the in-pipe and out-pipe are the same")
+ ErrExistingFileNotFifo = errors.New("the existing file is not a FIFO")
+)
+
+const (
+ pipeDuplexOpenReaderFlags = syscall.O_RDONLY | syscall.O_NONBLOCK
+ pipeDuplexOpenWriterFlags = syscall.O_WRONLY | syscall.O_NONBLOCK
+ pipeDuplexModeFlags = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP | syscall.S_IROTH
+ pipeDuplexPollInterval = 50 * time.Millisecond
+ pipeDuplex_PIPE_BUF = 4096
+)
+
+// Signal messages range from 1 to 255 & indicate control flow for the bytestream of the pipe.
+type SignalMessage byte
+
+const (
+ // DISCARD_PREVIOUS tells the receiver to discard previous partial work.
+ DiscardPrevious SignalMessage = 0x01
+)
+
+type OnMessage = func(msg []byte) error
+
+// Creates a named-pipe communication duplex. Creates a named-pipe communication duplex.
+// The reader end is responsible for creating the pipe.
+//
+// The layers are:
+// 1. Raw binary data over pipes
+// 2. Variable-length binary packets with COBS
+// 3. JSON-like values with Message Pack
+type PipeDuplex struct {
+ inPath string
+ outPath string
+
+ rawOutMu sync.Mutex
+ rawOut chan []byte
+
+ ctx context.Context
+ cancel context.CancelFunc
+ errg *errgroup.Group
+}
+
+func NewPipeDuplex(inPath, outPath string, onMessage OnMessage) (*PipeDuplex, error) {
+ // they must be different files
+ if inPath == outPath {
+ return nil, ErrInOutPipesAreSame
+ }
+ // pipes should only ever be created, and only by the reader (one-way operations)
+ if err := ensureFifoExists(inPath); err != nil {
+ return nil, err
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ errg, ctx := errgroup.WithContext(ctx)
+ p := &PipeDuplex{
+ inPath: inPath,
+ outPath: outPath,
+
+ rawOut: make(chan []byte, 128), // TODO: decide on size of this w/ constant??
+
+ ctx: ctx,
+ cancel: cancel,
+ errg: errg,
+ }
+ // Reader
+ p.errg.Go(func() error {
+ return p.pipeBufferReader(onMessage)
+ })
+
+ // Writer
+ p.errg.Go(func() error {
+ return p.pipeBufferWriter()
+ })
+
+ return p, nil
+}
+
+// Close stops all goroutines and waits for them to exit.
+func (p *PipeDuplex) Close() error {
+ p.cancel()
+
+ // this channel is exclusively written to via methods on this object handle, so it is its owner;
+ // owners must be the ones to close channels to avoid race conditions
+ defer func() {
+ // lock channel to avoid race conditions when closing
+ p.rawOutMu.Lock()
+ defer p.rawOutMu.Unlock()
+
+ close(p.rawOut)
+ }()
+
+ return p.errg.Wait()
+}
+
+// SendMessage MessagePack-encodes a "value" and enqueues it to the writer.
+func (p *PipeDuplex) SendMessage(msg []byte) error {
+ // lock channel to avoid race conditions when closing
+ p.rawOutMu.Lock()
+ defer p.rawOutMu.Unlock()
+
+ // send message bytes over outRaw channel
+ select {
+ case p.rawOut <- msg:
+ // TODO: could this trigger a race condition if calling Close() immediately after SendMessage()???
+ // should I lock p.rawOut w/ a mutex??
+ return nil
+ case <-p.ctx.Done():
+ return nil
+ }
+}
+
+func (p *PipeDuplex) InPath() string { return p.inPath }
+func (p *PipeDuplex) OutPath() string { return p.outPath }
+
+// ===== Private =====
+
+func ensureFifoExists(path string) error {
+ // try to make a file if one doesn't exist already
+ // TODO: add equivalent of `ensure_parent_directory_exists(path)` here !!!!!! <- may cause bugs w/out it???
+ if err := unix.Mkfifo(path, pipeDuplexModeFlags); err != nil {
+ if errno, ok := err.(unix.Errno); ok {
+ // misc error, do not handle
+ if errno != unix.EEXIST {
+ return err
+ }
+
+ // ensure the file exists is FIFO
+ fi, err := os.Stat(path)
+ if err != nil {
+ return err // misc error, do not handle
+ }
+ if fi.Mode()&fs.ModeNamedPipe == 0 {
+ return ErrExistingFileNotFifo
+ }
+ return nil
+ } else {
+ return err // misc error, do not handle
+ }
+ }
+ return nil
+}
+
+func (p *PipeDuplex) pipeBufferReader(onMessage OnMessage) error {
+ // open reader in nonblocking mode -> should not fail & immediately open;
+ // this marks when the writer process has "started"
+ fd, err := unix.Open(p.inPath, pipeDuplexOpenReaderFlags, pipeDuplexModeFlags)
+ if err != nil {
+ return err
+ }
+ defer unix.Close(fd)
+
+ // continually pull from the pipe and interpret messages as such:
+ // - all messages are separated/framed by NULL bytes (zero)
+ // - messages with >=2 bytes are COBS-encoded messages, because
+ // the smallest COBS-encoded message is 2 bytes
+ // - 1-byte messages are therefore to be treated as control signals
+ var buf []byte // accumulation buffer
+ for {
+ select { // check for kill-signal
+ case <-p.ctx.Done():
+ return nil
+ default:
+ }
+
+ // read available data (and try again if nothing)
+ data := make([]byte, pipeDuplex_PIPE_BUF)
+ n, err := unix.Read(fd, data)
+ if err != nil {
+ errno, ok := err.(unix.Errno)
+ if !ok || errno != unix.EAGAIN {
+ return err
+ }
+
+ // if there is a writer connected & the buffer is empty, this would block
+ // so we must consume this error gracefully and try again
+ time.Sleep(pipeDuplexPollInterval)
+ continue
+ }
+ if n == 0 {
+ time.Sleep(pipeDuplexPollInterval)
+ continue
+ }
+
+ // extend buffer with new data
+ buf = append(buf, data[:n]...)
+
+ // if there are no NULL bytes in the buffer, no new message has been formed
+ chunks := bytes.Split(buf, []byte{0x00})
+ if len(chunks) == 1 {
+ continue
+ }
+
+ // last chunk is always an unfinished message, so that becomes our new buffer;
+ // the rest should be decoded as either signals or COBS and put on queue
+ buf = chunks[len(chunks)-1]
+ for i := 0; i < len(chunks)-1; i++ {
+ chunk := chunks[i]
+
+ // ignore empty messages (they mean nothing)
+ if len(chunk) == 0 {
+ continue
+ }
+
+ // interpret 1-byte messages as signals (they indicate control-flow on messages)
+ if len(chunk) == 1 {
+ log.Printf("(reader): gotten control signal: %v", chunk[0])
+ // TODO: do some kind of stuff here??
+ continue
+ }
+
+ // interpret >=2 byte messages as COBS-encoded data (decode them)
+ decoded, err := cobs.Decode(chunk)
+ if err != nil {
+ return err
+ }
+
+ // call the callback to handle message
+ if err := onMessage(decoded); err != nil {
+ return err
+ }
+ }
+ }
+}
+
+func (p *PipeDuplex) pipeBufferWriter() error {
+ log.Printf("(writer): started")
+
+ // continually attempt to open FIFO for reading in nonblocking mode -> will error that:
+ // - ENOENT[2] No such file or directory: until a reader creates FIFO
+ // - ENXIO[6] No such device or address: until a reader opens FIFO
+ fd := -1
+ for {
+ select { // check for kill-signal
+ case <-p.ctx.Done():
+ return nil
+ default:
+ }
+
+ tempFd, err := unix.Open(p.outPath, pipeDuplexOpenWriterFlags, pipeDuplexModeFlags)
+ if err != nil {
+ if errno, ok := err.(unix.Errno); ok {
+ // misc error, do not handle
+ if !(errno == unix.ENOENT || errno == unix.ENXIO) {
+ return err
+ }
+
+ // try again if waiting for FIFO creation or reader-end opening
+ time.Sleep(pipeDuplexPollInterval)
+ continue
+ } else {
+ return err // misc error, do not handle
+ }
+ }
+ fd = tempFd
+ defer unix.Close(fd)
+
+ // ensure the file exists is FIFO
+ mode, err := lib.FstatGetMode(fd)
+ if err != nil {
+ return err // misc error, do not handle
+ }
+ if mode&fs.ModeNamedPipe == 0 {
+ return ErrExistingFileNotFifo
+ }
+
+ break // continue logic
+ }
+
+ // read bytes from rawOut & write them to pipe
+ for {
+ select {
+ case buf, ok := <-p.rawOut:
+ if !ok {
+ return nil
+ }
+ if err := p.writeData(fd, buf); err != nil {
+ return err
+ }
+ case <-p.ctx.Done():
+ return nil
+ }
+
+ }
+}
+
+func (p *PipeDuplex) writeData(fd int, buf []byte) error {
+ // COBS-encode the data & append NULL-byte to signify end-of-frame
+ buf, err := cobs.Encode(buf)
+ if err != nil {
+ return err
+ }
+ buf = append(buf, 0x00)
+ total := len(buf)
+ sent := 0
+
+ // begin transmission progress
+ for sent < total {
+ select { // check for kill-signal
+ case <-p.ctx.Done():
+ return nil
+ default:
+ }
+
+ // write & progress on happy path
+ written, err := unix.Write(fd, buf[sent:])
+ if err == nil {
+ sent += written
+ continue
+ }
+
+ // cast to OS error for propper handling
+ errno, ok := err.(unix.Errno)
+ if !ok {
+ return err // misc error, do not handle
+ }
+
+ // non-blocking pipe is full, wait a bit and retry
+ if errno == syscall.EAGAIN {
+ time.Sleep(pipeDuplexPollInterval)
+ continue
+ }
+
+ // reader disconnected -> handle failure-recovery by doing:
+ // 1. signal DISCARD_PREVIOUS to any reader
+ // 2. re-setting the progress & trying again
+ if errno == syscall.EPIPE {
+ if err := p.writeSignal(fd, DiscardPrevious); err != nil {
+ return err
+ }
+ sent = 0
+ continue
+ }
+
+ return err // misc error, do not handle
+ }
+ return nil
+}
+
+func (p *PipeDuplex) writeSignal(fd int, sig SignalMessage) error {
+ signalMessageLength := 2
+
+ // Turn signal-byte into message by terminating with NULL-byte
+ buf := []byte{byte(sig), 0x00}
+ lib.Assert(len(buf) == signalMessageLength, "this must never NOT be the case")
+
+ // attempt to write until successful
+ for {
+ select { // check for kill-signal
+ case <-p.ctx.Done():
+ return nil
+ default:
+ }
+
+ // small writes (e.g. 2 bytes) should be atomic as per Pipe semantics,
+ // meaning IF SUCCESSFUL: the number of bytes written MUST be exactly 2
+ written, err := unix.Write(fd, buf)
+ if err == nil {
+ lib.Assert(written == signalMessageLength, "this must never NOT be the case")
+ break
+ }
+
+ // cast to OS error for propper handling
+ errno, ok := err.(unix.Errno)
+ if !ok {
+ return err // misc error, do not handle
+ }
+
+ // wait a bit and retry if:
+ // - non-blocking pipe is full
+ // - the pipe is broken because of reader disconnection
+ if errno == syscall.EAGAIN || errno == syscall.EPIPE {
+ time.Sleep(pipeDuplexPollInterval)
+ continue
+ }
+
+ return err // misc error, do not handle
+ }
+ return nil
+}
diff --git a/networking/forwarder/lib/ipc/pipe_duplex_test.go b/networking/forwarder/lib/ipc/pipe_duplex_test.go
new file mode 100644
index 00000000..7cd87b2d
--- /dev/null
+++ b/networking/forwarder/lib/ipc/pipe_duplex_test.go
@@ -0,0 +1,85 @@
+//go:build unix
+
+package ipc
+
+import (
+ "log"
+ "os"
+ "testing"
+ "time"
+)
+
+func TestOneTwoThree(t *testing.T) {
+ // Avoid SIGPIPE killing the test if a writer outlives its reader.
+ // signal.Ignore(syscall.SIGPIPE) TODO: shoudn't sigpipe be handled by the error-code deep inside the duplex??
+
+ // Clean slate before/after.
+ onePath := "/tmp/one.pipe"
+ twoPath := "/tmp/two.pipe"
+ _ = os.Remove(onePath)
+ _ = os.Remove(twoPath)
+ defer os.Remove(onePath)
+ defer os.Remove(twoPath)
+
+ owner, err := NewPipeDuplex(
+ onePath, // in
+ twoPath, // out
+ func(m []byte) error { log.Printf("wow, owner got: [%v]%v", len(m), m); return nil },
+ )
+ if err != nil {
+ t.Fatalf("owner New failed: %v", err)
+ }
+
+ time.Sleep(1 * time.Second)
+
+ guest1, err := NewPipeDuplex(
+ twoPath, // in
+ onePath, // out
+ func(m []byte) error { log.Printf("wow, guest1 got: [%v]%v", len(m), m); return nil },
+ )
+ if err != nil {
+ t.Fatalf("guest1 New failed: %v", err)
+ }
+
+ if err := owner.SendMessage(make([]byte, 10)); err != nil {
+ t.Fatalf("owner SendMessage failed: %v", err)
+ }
+
+ // batch send
+ if err := guest1.SendMessage(make([]byte, 200)); err != nil {
+ t.Fatalf("guest1 SendMessage failed: %v", err)
+ }
+
+ time.Sleep(1 * time.Second)
+
+ if err := guest1.Close(); err != nil {
+ t.Fatalf("guest1 Close failed: %v", err)
+ }
+
+ if err := owner.SendMessage(make([]byte, 21)); err != nil {
+ t.Fatalf("owner SendMessage failed: %v", err)
+ }
+
+ guest2, err := NewPipeDuplex(
+ twoPath, // in
+ onePath, // out
+ func(m []byte) error { log.Printf("wow, guest2 got: [%v]%v", len(m), m); return nil },
+ )
+ if err != nil {
+ t.Fatalf("guest2 New failed: %v", err)
+ }
+
+ if err := guest2.SendMessage(make([]byte, 12)); err != nil {
+ t.Fatalf("guest2 SendMessage failed: %v", err)
+ }
+
+ time.Sleep(1 * time.Second)
+
+ if err := guest2.Close(); err != nil {
+ t.Fatalf("guest2 Close failed: %v", err)
+ }
+ if err := owner.Close(); err != nil {
+ t.Fatalf("owner Close failed: %v", err)
+ }
+ t.Fail()
+}
diff --git a/networking/forwarder/lib/libp2pext/dm/config.go b/networking/forwarder/lib/libp2pext/dm/config.go
new file mode 100644
index 00000000..9fdc9d3e
--- /dev/null
+++ b/networking/forwarder/lib/libp2pext/dm/config.go
@@ -0,0 +1,38 @@
+package dm
+
+import (
+ "context"
+
+ logging "github.com/ipfs/go-log/v2"
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/protocol"
+)
+
+type Config struct {
+ Host host.Host
+ Protocol protocol.ID
+ MessageHandler MessageHandler
+ Logger *logging.ZapEventLogger
+}
+
+type Option func(c *Config) error // TODO: add more options ??
+
+func WithHandler(h MessageHandler) Option {
+ return func(c *Config) error {
+ c.MessageHandler = h
+ return nil
+ }
+}
+func WithHandlerFunction(onMessage func(ctx context.Context, from peer.ID, msg []byte) error) Option {
+ return func(c *Config) error {
+ c.MessageHandler = &MessageHandlerBundle{OnMessageF: onMessage}
+ return nil
+ }
+}
+func WithLogger(l *logging.ZapEventLogger) Option {
+ return func(c *Config) error {
+ c.Logger = l
+ return nil
+ }
+}
diff --git a/networking/forwarder/lib/libp2pext/dm/dm.go b/networking/forwarder/lib/libp2pext/dm/dm.go
new file mode 100644
index 00000000..5cdba978
--- /dev/null
+++ b/networking/forwarder/lib/libp2pext/dm/dm.go
@@ -0,0 +1,57 @@
+package dm
+
+import (
+ "context"
+ "errors"
+
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/protocol"
+)
+
+const (
+ ServiceName = "libp2p.ext.dm/v1"
+ DmProtocol = protocol.ID("/dm/1.0.0")
+)
+
+var (
+ ErrMissingHandler = errors.New("the message handler is missing")
+)
+
+type MessageHandler interface {
+ OnMessage(ctx context.Context, from peer.ID, msg []byte) error
+}
+
+type MessageHandlerBundle struct {
+ OnMessageF func(ctx context.Context, from peer.ID, msg []byte) error
+}
+
+func (m *MessageHandlerBundle) OnMessage(ctx context.Context, from peer.ID, msg []byte) error {
+ return m.OnMessageF(ctx, from, msg)
+}
+
+type DirectMessenger interface {
+ Send(to peer.ID, msg []byte) error
+ Close() error
+}
+
+func NewDirectMessenger(h host.Host, opts ...Option) (DirectMessenger, error) {
+ cfg := &Config{
+ Host: h,
+ Protocol: DmProtocol,
+ Logger: logger,
+ }
+
+ // apply all configs
+ for _, o := range opts {
+ if err := o(cfg); err != nil {
+ return nil, err
+ }
+ }
+ if cfg.MessageHandler == nil {
+ return nil, ErrMissingHandler
+ }
+
+ // create DM from config
+ return newDirectMessenger(cfg)
+}
diff --git a/networking/forwarder/lib/libp2pext/dm/dm_test.go b/networking/forwarder/lib/libp2pext/dm/dm_test.go
new file mode 100644
index 00000000..afa6cf02
--- /dev/null
+++ b/networking/forwarder/lib/libp2pext/dm/dm_test.go
@@ -0,0 +1,88 @@
+package dm
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "log"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/peer"
+ libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
+
+ "github.com/libp2p/go-libp2p"
+)
+
+func genPriv(t *testing.T, seed [32]byte) crypto.PrivKey {
+ priv, _, err := crypto.GenerateEd25519Key(bytes.NewReader(seed[:]))
+ if err != nil {
+ t.Fatalf("failed generating key from seed %v: %v", seed, err)
+ }
+ return priv
+}
+
+func createTestHost(t *testing.T, name string, opts ...Option) (host.Host, DirectMessenger) {
+ // generate key
+ seed := sha256.Sum256([]byte(name))
+ id := genPriv(t, seed)
+
+ // create host
+ h, err := libp2p.New(
+ libp2p.Identity(id),
+ libp2p.Transport(libp2pquic.NewTransport),
+ libp2p.ListenAddrStrings(
+ "/ip4/0.0.0.0/udp/0/quic-v1",
+ ),
+ )
+ if err != nil {
+ t.Fatalf("failed creating test host '%v': %v", name, err)
+ }
+
+ // configure direct messaging
+ dmOpts := []Option{WithHandler(&MessageHandlerBundle{
+ OnMessageF: func(ctx context.Context, from peer.ID, msg []byte) error {
+ log.Printf("[%v]<-[%v]: [%v]%v", name, from, len(msg), msg)
+ return nil
+ },
+ })}
+ dmOpts = append(dmOpts, opts...)
+ dm, err := NewDirectMessenger(h, dmOpts...)
+ if err != nil {
+ t.Fatalf("failed creating test DM manager for host '%v': %v", name, err)
+ }
+
+ return h, dm
+}
+
+func createConnection(t *testing.T, p1, p2 host.Host) {
+ ctx := context.Background()
+ if err := p1.Connect(ctx, p2.Peerstore().PeerInfo(p2.ID())); err != nil {
+ t.Fatalf("failed connecting '%v' to '%v': %v", p1.ID(), p2.ID(), err)
+ }
+}
+
+func TestJsonEncoder(t *testing.T) {
+ peer1, dm1 := createTestHost(t, "peer 1")
+ defer dm1.Close()
+ defer peer1.Close()
+
+ peer2, dm2 := createTestHost(t, "peer 2")
+ defer dm2.Close()
+ defer peer2.Close()
+
+ createConnection(t, peer1, peer2)
+
+ if err := dm1.Send(peer2.ID(), make([]byte, 10)); err != nil {
+ t.Fatalf("dm1 Send failed: %v", err)
+ }
+
+ // big send
+ if err := dm2.Send(peer1.ID(), make([]byte, 10_000)); err != nil {
+ t.Fatalf("dm2 Send failed: %v", err)
+ }
+ time.Sleep(500 * time.Millisecond)
+ t.Fail()
+}
diff --git a/networking/forwarder/lib/libp2pext/dm/internal.go b/networking/forwarder/lib/libp2pext/dm/internal.go
new file mode 100644
index 00000000..24b7dff3
--- /dev/null
+++ b/networking/forwarder/lib/libp2pext/dm/internal.go
@@ -0,0 +1,151 @@
+package dm
+
+import (
+ "context"
+ "encoding/binary"
+ "io"
+ "lib"
+ "sync"
+
+ logging "github.com/ipfs/go-log/v2"
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/protocol"
+ "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
+)
+
+const (
+ uint64NumBytes = 8
+)
+
+var (
+ logger = logging.Logger(ServiceName)
+)
+
+type directMessenger struct {
+ ctx context.Context
+ cancel func()
+
+ h host.Host
+ pid protocol.ID
+ handler MessageHandler
+ log *logging.ZapEventLogger
+
+ scope network.ResourceScopeSpan
+ notifiee network.Notifiee
+
+ mx sync.Mutex
+ closed bool
+}
+
+func newDirectMessenger(cfg *Config) (*directMessenger, error) {
+ ctx, cancel := context.WithCancel(context.Background())
+ dm := &directMessenger{
+ ctx: ctx,
+ cancel: cancel,
+
+ h: cfg.Host,
+ pid: cfg.Protocol,
+ handler: cfg.MessageHandler,
+ log: cfg.Logger,
+ }
+
+ // get a scope for memory reservations at service level
+ err := dm.h.Network().ResourceManager().ViewService(ServiceName,
+ func(s network.ServiceScope) error {
+ var err error
+ dm.scope, err = s.BeginSpan()
+ return err
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ dm.h.SetStreamHandler(dm.pid, dm.handleStream)
+ dm.notifiee = &network.NotifyBundle{} // TODO: add handler funcions in the future if so needed??
+ dm.h.Network().Notify(dm.notifiee)
+
+ return dm, nil
+}
+
+func (dm *directMessenger) Close() error {
+ dm.mx.Lock()
+ if !dm.closed {
+ dm.closed = true
+ dm.mx.Unlock()
+
+ dm.h.RemoveStreamHandler(proto.ProtoIDv2Hop)
+ dm.h.Network().StopNotify(dm.notifiee)
+ defer dm.scope.Done()
+ dm.cancel()
+ return nil
+ }
+ dm.mx.Unlock()
+ return nil
+}
+
+func (dm *directMessenger) Send(p peer.ID, msg []byte) error {
+ dm.log.Infof("outgoing DM stream to: %s", p)
+
+ // create new stream
+ s, err := dm.h.NewStream(dm.ctx, p, dm.pid)
+ if err != nil {
+ return err
+ }
+ defer s.Close()
+
+ // grab length if byte-buffer and encode it as big-endian
+ mLen := len(msg)
+ buf := make([]byte, uint64NumBytes, uint64NumBytes+mLen) // allocate enough capacity
+ binary.BigEndian.PutUint64(buf, uint64(mLen))
+ buf = append(buf, msg...)
+ lib.Assert(len(buf) == uint64NumBytes+mLen, "literally what????")
+
+ // write to stream & handle any potential errors
+ if _, err := s.Write(buf); err != nil {
+ dm.log.Debugf("error writing message to DM service stream: %s", err)
+ s.Reset()
+ return err
+ }
+
+ _ = s.CloseWrite() // signal EOF to caller if half-close is supported
+ return nil
+}
+
+func (dm *directMessenger) handleStream(s network.Stream) {
+ dm.log.Infof("incoming DM stream from: %s", s.Conn().RemotePeer())
+
+ defer s.Close()
+
+ // attach scope to this service (for scoped capacity allocation reasons)
+ if err := s.Scope().SetService(ServiceName); err != nil {
+ dm.log.Debugf("error attaching stream to DM service: %s", err)
+ s.Reset()
+ return
+ }
+
+ // read big-endian length bytes & decode
+ buf := make([]byte, uint64NumBytes)
+ if _, err := io.ReadFull(s, buf); err != nil {
+ dm.log.Debugf("error reading message length from DM service stream: %s", err)
+ s.Reset()
+ return
+ }
+ mLen := binary.BigEndian.Uint64(buf)
+
+ // read rest of message & call OnMessage callback
+ buf = make([]byte, mLen)
+ if _, err := io.ReadFull(s, buf); err != nil {
+ dm.log.Debugf("error reading message body from DM service stream: %s", err)
+ s.Reset()
+ return
+ }
+ if err := dm.handler.OnMessage(dm.ctx, s.Conn().RemotePeer(), buf); err != nil {
+ dm.log.Debugf("error handling incoming message from DM service stream: %s", err)
+ s.Reset()
+ return
+ }
+
+ _ = s.CloseWrite() // signal EOF to caller if half-close is supported
+}
diff --git a/networking/forwarder/lib/util.go b/networking/forwarder/lib/util.go
new file mode 100644
index 00000000..879b9ba3
--- /dev/null
+++ b/networking/forwarder/lib/util.go
@@ -0,0 +1,52 @@
+package lib
+
+import (
+ "log"
+ "os"
+ "syscall"
+
+ "golang.org/x/sys/unix"
+)
+
+func Assert(b bool, msg string) {
+ if !b {
+ log.Panic(msg)
+ }
+}
+
+func FstatGetMode(fd int) (os.FileMode, error) {
+ // perform fstat syscall
+ var sys unix.Stat_t = unix.Stat_t{}
+ if err := unix.Fstat(fd, &sys); err != nil {
+ return 0, err
+ }
+
+ // reconstruct FileMode from sys-struct; SEE: https://github.com/golang/go/blob/5a56d8848b4ffb79c5ccc11ec6fa01823a91aaf8/src/os/stat_linux.go#L17
+ mode := os.FileMode(sys.Mode & 0777)
+ switch sys.Mode & syscall.S_IFMT {
+ case syscall.S_IFBLK:
+ mode |= os.ModeDevice
+ case syscall.S_IFCHR:
+ mode |= os.ModeDevice | os.ModeCharDevice
+ case syscall.S_IFDIR:
+ mode |= os.ModeDir
+ case syscall.S_IFIFO:
+ mode |= os.ModeNamedPipe
+ case syscall.S_IFLNK:
+ mode |= os.ModeSymlink
+ case syscall.S_IFREG:
+ // nothing to do
+ case syscall.S_IFSOCK:
+ mode |= os.ModeSocket
+ }
+ if sys.Mode&syscall.S_ISGID != 0 {
+ mode |= os.ModeSetgid
+ }
+ if sys.Mode&syscall.S_ISUID != 0 {
+ mode |= os.ModeSetuid
+ }
+ if sys.Mode&syscall.S_ISVTX != 0 {
+ mode |= os.ModeSticky
+ }
+ return mode, nil
+}
diff --git a/pyproject.toml b/pyproject.toml
index 20f8b5b6..d43868ef 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,6 +30,7 @@ dependencies = [
"mlx-lm @ https://github.com/ml-explore/mlx-lm.git",
"psutil>=7.0.0",
"transformers>=4.55.2",
+ "cobs>=1.2.2",
]
[project.scripts]
diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml
index 8d10af64..54bf6702 100644
--- a/scripts/pyproject.toml
+++ b/scripts/pyproject.toml
@@ -5,10 +5,13 @@ description = "Scripts for the Exo project"
readme = "README.md"
requires-python = ">=3.13"
dependencies = [
- "shared",
"huggingface_hub>=0.33.4",
+ "exo"
]
[build-system]
requires = ["uv_build>=0.8.9,<0.9.0"]
build-backend = "uv_build"
+
+[tool.uv.sources]
+exo = { workspace = true }
\ No newline at end of file
diff --git a/scripts/src/exo_scripts/read_events.py b/scripts/src/exo_scripts/read_events.py
index f8da5679..68fc9398 100644
--- a/scripts/src/exo_scripts/read_events.py
+++ b/scripts/src/exo_scripts/read_events.py
@@ -1,9 +1,10 @@
+# pyright: reportAny=false
+
import asyncio
import curses
import time
import json
import argparse
-import textwrap
import sys
from logging import Logger
from typing import List, Optional, Any, Sequence, Tuple
@@ -27,13 +28,17 @@ WORKER_EVENT_TYPES = {
'RunnerStatusUpdated', 'RunnerDeleted'
}
+
async def init_db() -> None:
global event_log_manager
event_log_manager = EventLogManager(EventLogConfig(), logger)
await event_log_manager.initialize()
+
async def get_events_since(since: int) -> Sequence[EventFromEventLog[Event]]:
- return await event_log_manager.global_events.get_events_since(since) # type: ignore[attr-defined, return-value]
+ assert event_log_manager is not None
+ return await event_log_manager.global_events.get_events_since(since)
+
async def load_all_events() -> List[EventFromEventLog[Event]]:
events: List[EventFromEventLog[Event]] = []
@@ -46,6 +51,7 @@ async def load_all_events() -> List[EventFromEventLog[Event]]:
since += len(new_events)
return events
+
def compute_states(events: List[EventFromEventLog[Event]]) -> List[State]:
states: List[State] = [State()]
state = states[0]
@@ -54,12 +60,15 @@ def compute_states(events: List[EventFromEventLog[Event]]) -> List[State]:
states.append(state)
return states
+
def print_event(event: EventFromEventLog[Event]) -> None:
event_type_name = type(event.event).__name__
event_type = event_type_name.replace('_', ' ').title()
- attributes = ', '.join(f"{key}={value!r}" for key, value in vars(event.event).items())
+ attributes = ', '.join(f"{key}={value!r}" for key,
+ value in vars(event.event).items())
print(f"[{event.idx_in_log}] {event_type}: {attributes}")
+
async def non_tui_mode() -> None:
await init_db()
events = await load_all_events()
@@ -67,7 +76,8 @@ async def non_tui_mode() -> None:
final_state = states[-1]
if worker_mode:
- filtered_events = [e for e in events if type(e.event).__name__ in WORKER_EVENT_TYPES]
+ filtered_events = [e for e in events if type(
+ e.event).__name__ in WORKER_EVENT_TYPES]
events = filtered_events
# Recompute states? But states are cumulative, so perhaps just print filtered events and full state, or filter state too.
state_dict = json.loads(final_state.model_dump_json())
@@ -88,7 +98,9 @@ async def non_tui_mode() -> None:
for event in events:
print_event(event)
-async def update_events(wrapped_events: List[EventFromEventLog[Event]], states: List[State], filtered_indices: Optional[List[int]] = None) -> bool:
+
+async def update_events(wrapped_events: List[EventFromEventLog[Event]], states: List[State],
+ filtered_indices: Optional[List[int]] = None) -> bool:
last_since = len(wrapped_events)
new_wrapped = await get_events_since(last_since)
if new_wrapped:
@@ -105,6 +117,7 @@ async def update_events(wrapped_events: List[EventFromEventLog[Event]], states:
return True
return False
+
def draw_state(win: Any, state: State, height: int, width: int, worker_mode: bool, state_scroll: int) -> int:
win.clear()
state_dict = json.loads(state.model_dump_json())
@@ -142,11 +155,13 @@ def draw_state(win: Any, state: State, height: int, width: int, worker_mode: boo
value_str = stripped[end_key + 3:]
if value_str.startswith('"'):
color = 2
- elif value_str.replace('.', '', 1).isdigit() or (value_str.startswith('-') and value_str[1:].replace('.', '', 1).isdigit()):
+ elif value_str.replace('.', '', 1).isdigit() or (
+ value_str.startswith('-') and value_str[1:].replace('.', '', 1).isdigit()):
color = 4
elif value_str in ['true', 'false', 'null']:
color = 5
- elif value_str.startswith('{') or value_str.startswith('[') or value_str.startswith('}') or value_str.startswith(']'):
+ elif value_str.startswith('{') or value_str.startswith('[') or value_str.startswith(
+ '}') or value_str.startswith(']'):
color = 0
else:
color = 0
@@ -158,6 +173,7 @@ def draw_state(win: Any, state: State, height: int, width: int, worker_mode: boo
win.refresh()
return current_scroll
+
def get_event_pairs(event: EventFromEventLog[Event]) -> List[Tuple[str, int]]:
pairs: List[Tuple[str, int]] = []
idx_str = f"[{event.idx_in_log}] "
@@ -186,6 +202,7 @@ def get_event_pairs(event: EventFromEventLog[Event]) -> List[Tuple[str, int]]:
pairs.append((v_str, color))
return pairs
+
def calculate_event_lines(pairs: List[Tuple[str, int]], win_width: int, subsequent_indent: int) -> int:
lines = 1
x = 0
@@ -201,7 +218,9 @@ def calculate_event_lines(pairs: List[Tuple[str, int]], win_width: int, subseque
x = subsequent_indent
return lines
-def render_event(win: Any, start_y: int, pairs: List[Tuple[str, int]], is_bold: bool, win_width: int, subsequent_indent: int) -> int:
+
+def render_event(win: Any, start_y: int, pairs: List[Tuple[str, int]], is_bold: bool, win_width: int,
+ subsequent_indent: int) -> int:
y = start_y
x = 0
for text, color in pairs:
@@ -226,6 +245,7 @@ def render_event(win: Any, start_y: int, pairs: List[Tuple[str, int]], is_bold:
y += 1
return y
+
def draw_events(win: Any, events_list: List[EventFromEventLog[Event]], current_events: int, height: int) -> None:
win.clear()
if len(events_list) == 0:
@@ -236,7 +256,8 @@ def draw_events(win: Any, events_list: List[EventFromEventLog[Event]], current_e
current_event = events_list[current_events]
current_pairs = get_event_pairs(current_event)
subsequent_indent = len(f"[{current_event.idx_in_log}] ")
- lines_current = calculate_event_lines(current_pairs, win_width, subsequent_indent)
+ lines_current = calculate_event_lines(
+ current_pairs, win_width, subsequent_indent)
if lines_current > height:
render_event(win, 0, current_pairs, True, win_width, subsequent_indent)
win.refresh()
@@ -313,12 +334,15 @@ def draw_events(win: Any, events_list: List[EventFromEventLog[Event]], current_e
win.refresh()
+
def draw_status(win: Any, realtime: bool, current: int, total_events: int) -> None:
win.clear()
mode = "Realtime" if realtime else "Timetravel"
- win.addstr(0, 0, f"Mode: {mode} | Current event: {current} / {total_events} | Arrows: navigate events, [/]: scroll state, g: goto, r: toggle realtime, q: quit")
+ win.addstr(0, 0,
+ f"Mode: {mode} | Current event: {current} / {total_events} | Arrows: navigate events, [/]: scroll state, g: goto, r: toggle realtime, q: quit")
win.refresh()
+
def get_input(stdscr: Any, prompt: str) -> str:
curses.echo()
stdscr.addstr(0, 0, prompt)
@@ -327,6 +351,7 @@ def get_input(stdscr: Any, prompt: str) -> str:
curses.noecho()
return input_str
+
def get_key(win: Any) -> Any:
ch = win.getch()
if ch == -1:
@@ -369,6 +394,7 @@ def get_key(win: Any) -> Any:
return 'CTRL_DOWN'
return ch
+
def tui(stdscr: Any) -> None:
curses.start_color()
curses.init_pair(1, curses.COLOR_BLUE, curses.COLOR_BLACK)
@@ -390,8 +416,10 @@ def tui(stdscr: Any) -> None:
current_filtered: int = -1
current: int = -1
if worker_mode:
- filtered_indices = [i for i in range(len(wrapped_events)) if type(wrapped_events[i].event).__name__ in WORKER_EVENT_TYPES]
- current_filtered = len(filtered_indices) - 1 if filtered_indices else -1
+ filtered_indices = [i for i in range(len(wrapped_events)) if
+ type(wrapped_events[i].event).__name__ in WORKER_EVENT_TYPES]
+ current_filtered = len(filtered_indices) - \
+ 1 if filtered_indices else -1
else:
current = len(wrapped_events) - 1 if wrapped_events else -1
@@ -407,7 +435,8 @@ def tui(stdscr: Any) -> None:
pane_width = width // 2
state_win = curses.newwin(pane_height, pane_width, 0, 0)
- events_win = curses.newwin(pane_height, width - pane_width, 0, pane_width)
+ events_win = curses.newwin(
+ pane_height, width - pane_width, 0, pane_width)
status_win = curses.newwin(status_height, width, pane_height, 0)
if worker_mode:
@@ -421,10 +450,12 @@ def tui(stdscr: Any) -> None:
current_events = current
state_idx = current_original + 1 if current_original >= 0 else 0
- state_scroll = draw_state(state_win, states[state_idx], pane_height, pane_width, worker_mode, state_scroll)
+ state_scroll = draw_state(
+ state_win, states[state_idx], pane_height, pane_width, worker_mode, state_scroll)
draw_events(events_win, events_list, current_events, pane_height)
total_events = len(wrapped_events) - 1 if wrapped_events else -1
- draw_status(status_win, realtime, current_original if worker_mode else current, total_events)
+ draw_status(status_win, realtime,
+ current_original if worker_mode else current, total_events)
key = get_key(stdscr)
if key != -1:
@@ -439,13 +470,16 @@ def tui(stdscr: Any) -> None:
else:
current = max(0, current - 5)
elif key == curses.KEY_DOWN:
- if worker_mode and current_filtered < len(filtered_indices) - 1: # type: ignore[arg-type]
+ assert filtered_indices is not None
+ if worker_mode and current_filtered < len(filtered_indices) - 1:
current_filtered += 1
elif not worker_mode and current < len(wrapped_events) - 1:
current += 1
elif key == 'CTRL_DOWN':
+ assert filtered_indices is not None
if worker_mode:
- current_filtered = min(len(filtered_indices) - 1, current_filtered + 5) # type: ignore[arg-type]
+ current_filtered = min(
+ len(filtered_indices) - 1, current_filtered + 5)
else:
current = min(len(wrapped_events) - 1, current + 5)
elif key == ord('['):
@@ -457,10 +491,13 @@ def tui(stdscr: Any) -> None:
elif key == ord('r'):
realtime = not realtime
if realtime:
+ assert filtered_indices is not None
if worker_mode:
- current_filtered = len(filtered_indices) - 1 if filtered_indices else -1 # type: ignore[arg-type]
+ current_filtered = len(
+ filtered_indices) - 1 if filtered_indices else -1
else:
- current = len(wrapped_events) - 1 if wrapped_events else -1
+ current = len(wrapped_events) - \
+ 1 if wrapped_events else -1
state_scroll = 0
elif key == ord('g'):
stdscr.timeout(-1) # block for input
@@ -487,18 +524,23 @@ def tui(stdscr: Any) -> None:
status_win.refresh()
if realtime and time.time() - last_update > update_interval:
- updated = asyncio.run(update_events(wrapped_events, states, filtered_indices if worker_mode else None))
+ updated = asyncio.run(update_events(
+ wrapped_events, states, filtered_indices if worker_mode else None))
if updated:
+ assert filtered_indices is not None
if worker_mode:
- current_filtered = len(filtered_indices) - 1 # type: ignore[arg-type]
+ current_filtered = len(filtered_indices) - 1
else:
current = len(wrapped_events) - 1
state_scroll = 0
last_update = time.time()
+
if __name__ == "__main__":
- parser = argparse.ArgumentParser(description='Read and display events from the event log')
- parser.add_argument('--worker', action='store_true', help='Only show worker-related events (task, streaming, instance, runner status)')
+ parser = argparse.ArgumentParser(
+ description='Read and display events from the event log')
+ parser.add_argument('--worker', action='store_true',
+ help='Only show worker-related events (task, streaming, instance, runner status)')
args = parser.parse_args()
worker_mode = args.worker
@@ -513,4 +555,4 @@ if __name__ == "__main__":
print("Error: Could not find terminal. Falling back to non-TUI mode.")
asyncio.run(non_tui_mode())
else:
- raise
\ No newline at end of file
+ raise
diff --git a/scripts/watch-pull-restart.py b/scripts/watch-pull-restart.py
new file mode 100755
index 00000000..aad5c0b2
--- /dev/null
+++ b/scripts/watch-pull-restart.py
@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+
+"""
+watch-pull-restart.py — Unix-only
+
+Runs a command, periodically checks git upstream, pulls if upstream is ahead,
+and gracefully restarts the command. Watcher logs go to STDERR; your app's
+output goes straight to the console (STDOUT/STDERR).
+
+Assumptions:
+ - current branch tracks an upstream (i.e., @{u} exists)
+ - pulls must be fast-forward (remote-ahead workflow)
+
+Arguments:
+ - cmd: Command to run/manage (e.g. './run.sh' or 'python -m app').
+ - restart-cmd: Optional hook to run after a successful pull (e.g., systemctl restart).
+ - sleep-secs: Poll interval while up-to-date.
+ - grace-secs: Seconds to wait after SIGTERM before SIGKILL.
+ - debounce-secs: Coalesce multiple pulls before restart.
+
+Usage:
+ ./watch-pull-restart.py --cmd "./run.sh" --sleep-secs 1
+ ./watch-pull-restart.py --cmd "python -m app" --restart-cmd "systemctl --user restart myapp"
+ ./watch-pull-restart.py --restart-cmd "systemctl --user restart myapp" # no managed child; only trigger hook
+"""
+import argparse
+import os
+import signal
+import subprocess
+import sys
+import time
+from types import FrameType
+from typing import Optional
+
+
+# ---------- logging helpers (to STDERR) ----------
+def log(msg: str):
+ sys.stderr.write(msg.rstrip() + "\n")
+ sys.stderr.flush()
+
+
+def sep(title: str = ""):
+ """Big visual separator for state transitions (to STDERR)."""
+ sys.stderr.write("\n\n")
+ if title:
+ sys.stderr.write(f"===== [watch] {title} =====\n")
+ else:
+ sys.stderr.write("===== [watch] =====\n")
+ sys.stderr.flush()
+
+
+def run_capture(cmd: str, check: bool = True) -> subprocess.CompletedProcess[str]:
+ """Run and capture output; for git plumbing."""
+ return subprocess.run(
+ cmd,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ check=check,
+ )
+
+
+# ---------- shell helpers ----------
+def is_up_to_date() -> bool:
+ subprocess.run("git fetch --quiet",
+ shell=True) # Quiet fetch; ignore network errors (we'll just try again next tick)
+ try:
+ current = run_capture("git rev-parse HEAD", check=True).stdout.strip()
+ upstream = run_capture("git rev-parse @{u}", check=True).stdout.strip()
+ return current == upstream
+ except subprocess.CalledProcessError:
+ return True # No upstream or other git error; treat as up-to-date to avoid thrash
+
+
+def pull_ff_only() -> bool:
+ """Returns True if pull applied changes, False if already up-to-date."""
+ try:
+ cp = run_capture("git pull --ff-only --no-rebase", check=True)
+ return "Already up to date" not in cp.stdout and cp.returncode == 0 # Git prints "Already up to date." on no-op; cheap heuristic
+ except subprocess.CalledProcessError as e:
+ log("[watch] git pull failed:")
+ if e.stdout: # pyright: ignore[reportAny]
+ log(e.stdout) # pyright: ignore[reportAny]
+ if e.stderr: # pyright: ignore[reportAny]
+ log(e.stderr) # pyright: ignore[reportAny]
+ return False
+
+
+# ---------- managed processes ----------
+class ManagedProc:
+ def __init__(self, cmd: Optional[str], grace_secs: float):
+ self.cmd = cmd
+ self.grace = grace_secs
+ self.child: Optional[subprocess.Popen[bytes]] = None
+
+ def start(self):
+ if not self.cmd:
+ return
+ if self.child and self.child.poll() is None:
+ return
+ sep("starting main cmd")
+ log(f"[watch] starting: {self.cmd}")
+ # New process group so we can signal the entire tree (shell + children)
+ self.child = subprocess.Popen(
+ self.cmd,
+ shell=True, # allow shell features in --cmd
+ stdout=None, # inherit parent's stdout (your app prints normally)
+ stderr=None, # inherit parent's stderr
+ stdin=None,
+ preexec_fn=os.setsid, # create new session (PGID == child PID)
+ )
+
+ def stop_gracefully(self):
+ if not self.child:
+ return
+ if self.child.poll() is not None:
+ self.child = None
+ return
+
+ sep("stopping main cmd (SIGTERM)")
+ try:
+ os.killpg(self.child.pid, signal.SIGTERM)
+ except ProcessLookupError:
+ pass
+
+ deadline = time.time() + self.grace
+ while time.time() < deadline:
+ if self.child.poll() is not None:
+ self.child = None
+ return
+ time.sleep(0.1)
+
+ sep("main cmd unresponsive; SIGKILL")
+ try:
+ os.killpg(self.child.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ self.child = None
+
+ def forward_signal(self, sig: int):
+ if not self.child or self.child.poll() is not None:
+ return
+ try:
+ os.killpg(self.child.pid, sig)
+ except ProcessLookupError:
+ pass
+
+
+class OneShotHook:
+ """
+ One-shot hook command (e.g., systemctl restart).
+ Runs to completion with inherited stdio so its output is visible.
+ """
+
+ def __init__(self, cmd: Optional[str], grace_secs: float):
+ self.cmd = cmd
+ self.grace = grace_secs
+ self.child: Optional[subprocess.Popen[bytes]] = None
+
+ def run(self) -> int:
+ if not self.cmd:
+ return 0
+ sep("running restart hook")
+ log(f"[watch] hook: {self.cmd}")
+ self.child = subprocess.Popen(
+ self.cmd,
+ shell=True,
+ stdout=None, # inherit stdio
+ stderr=None,
+ stdin=None,
+ preexec_fn=os.setsid,
+ )
+ # Wait with grace/kill if needed (rare for hooks, but symmetric)
+ deadline = time.time() + self.grace
+ while True:
+ rc = self.child.poll()
+ if rc is not None:
+ self.child = None
+ return rc
+ if time.time() > deadline:
+ sep("hook exceeded grace; SIGKILL")
+ try:
+ os.killpg(self.child.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ self.child = None
+ return 137 # killed
+ time.sleep(0.1)
+
+ def forward_signal(self, sig: int):
+ if not self.child or self.child.poll() is not None:
+ return
+ try:
+ os.killpg(self.child.pid, sig)
+ except ProcessLookupError:
+ pass
+
+
+# ---------- main loop ----------
+def main():
+ # CMD commands
+ ap = argparse.ArgumentParser(description="Auto-pull & restart on upstream changes (Unix).")
+ ap.add_argument("--cmd", help="Command to run/manage (e.g. './run.sh' or 'python -m app').")
+ ap.add_argument("--restart-cmd", help="Optional hook to run after a successful pull (e.g., systemctl restart).")
+ ap.add_argument("--sleep-secs", type=float, default=0.5, help="Poll interval while up-to-date.")
+ ap.add_argument("--grace-secs", type=float, default=5.0, help="Seconds to wait after SIGTERM before SIGKILL.")
+ ap.add_argument("--debounce-secs", type=float, default=0.5, help="Coalesce multiple pulls before restart.")
+ args = ap.parse_args()
+
+ # get CMD command values
+ cmd = args.cmd # pyright: ignore[reportAny]
+ assert cmd is None or isinstance(cmd, str)
+ restart_cmd = args.restart_cmd # pyright: ignore[reportAny]
+ assert cmd is None or isinstance(restart_cmd, str)
+ sleep_secs = args.sleep_secs # pyright: ignore[reportAny]
+ assert sleep_secs is not None and isinstance(sleep_secs, float)
+ grace_secs = args.grace_secs # pyright: ignore[reportAny]
+ assert sleep_secs is not None and isinstance(grace_secs, float)
+ debounce_secs = args.debounce_secs # pyright: ignore[reportAny]
+ assert sleep_secs is not None and isinstance(debounce_secs, float)
+
+ # start managed proc
+ proc = ManagedProc(cmd, grace_secs)
+ hook = OneShotHook(restart_cmd, grace_secs)
+
+ # signal handling for graceful exit
+ exiting = {"flag": False}
+
+ def _handle(sig_num: int, _frame: Optional[FrameType]):
+ sep(f"received signal {sig_num}; exiting")
+ exiting["flag"] = True
+ proc.forward_signal(sig_num)
+ hook.forward_signal(sig_num)
+
+ signal.signal(signal.SIGINT, _handle)
+ signal.signal(signal.SIGTERM, _handle)
+
+ # Initial start (if managing a process)
+ proc.start()
+
+ pending_restart = False
+ last_change = 0.0
+ while not exiting["flag"]:
+ try:
+ if not is_up_to_date():
+ sep("upstream ahead; pulling")
+ changed = pull_ff_only()
+ if changed:
+ last_change = time.time()
+ pending_restart = True
+
+ # handle debounce window
+ if pending_restart and (time.time() - last_change) >= debounce_secs:
+ # Optional hook first
+ if restart_cmd:
+ rc = hook.run()
+ if rc != 0:
+ sep(f"hook exited with {rc}")
+ # Then bounce managed process
+ if cmd:
+ proc.stop_gracefully()
+ proc.start()
+ pending_restart = False
+ sep("restart cycle complete")
+
+ # keep the child alive if it crashed without a pull
+ if cmd and (proc.child is None or proc.child.poll() is not None):
+ sep("main cmd exited; restarting")
+ proc.start()
+
+ time.sleep(sleep_secs)
+ except Exception as e:
+ sep("loop error")
+ log(f"[watch] {e}")
+ time.sleep(2.0)
+
+ # graceful shutdown on exit
+ proc.stop_gracefully()
+ sep("bye")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/exo/engines/mlx/auto_parallel.py b/src/exo/engines/mlx/auto_parallel.py
index 2e2589fa..383cb8c2 100644
--- a/src/exo/engines/mlx/auto_parallel.py
+++ b/src/exo/engines/mlx/auto_parallel.py
@@ -1,7 +1,7 @@
from typing import Protocol, cast, override
import mlx.core as mx
-import mlx.nn as nn
+import mlx.nn as nn # pyright: ignore[reportMissingTypeStubs]
from exo.shared.types.worker.shards import PipelineShardMetadata
diff --git a/src/exo/engines/mlx/utils_mlx.py b/src/exo/engines/mlx/utils_mlx.py
index 60a21e30..955cbb88 100644
--- a/src/exo/engines/mlx/utils_mlx.py
+++ b/src/exo/engines/mlx/utils_mlx.py
@@ -5,14 +5,14 @@ import resource
from asyncio import AbstractEventLoop
from typing import Any, Callable
-import mlx.core as mx
-import mlx.nn as nn
from mlx_lm.generate import stream_generate # type: ignore
from mlx_lm.sample_utils import make_sampler
from mlx_lm.tokenizer_utils import TokenizerWrapper, load_tokenizer # type: ignore
from mlx_lm.utils import load_model # type: ignore
from pydantic import RootModel
+import mlx.core as mx
+import mlx.nn as nn # pyright: ignore[reportMissingTypeStubs]
from exo.engines.mlx.auto_parallel import auto_parallel
from exo.shared.types.api import ChatCompletionMessage
from exo.shared.types.common import Host
@@ -117,8 +117,10 @@ async def apply_chat_template(
formatted_messages = []
for message in messages_dicts:
filtered_message: dict[str, Any] = {
- k: v for k, v in message.items() if v is not None
- } # type: ignore
+ k: v
+ for k, v in message.items() # pyright: ignore[reportAny]
+ if v is not None
+ }
# Verify we have required fields
if "role" not in filtered_message:
diff --git a/src/exo/master/forwarder_supervisor.py b/src/exo/master/forwarder_supervisor.py
index 50798c8a..a1fb6120 100644
--- a/src/exo/master/forwarder_supervisor.py
+++ b/src/exo/master/forwarder_supervisor.py
@@ -113,6 +113,7 @@ class ForwarderSupervisor:
str(self._binary_path),
"--events-db",
str(EXO_WORKER_EVENT_DB),
+ # pair arguments
f"{pairs}",
stdout=None,
stderr=None,
diff --git a/src/exo/master/main.py b/src/exo/master/main.py
index c0709db2..e7f982cb 100644
--- a/src/exo/master/main.py
+++ b/src/exo/master/main.py
@@ -14,6 +14,7 @@ from exo.shared.apply import apply
from exo.shared.db.sqlite.config import EventLogConfig
from exo.shared.db.sqlite.connector import AsyncSQLiteEventStorage
from exo.shared.db.sqlite.event_log_manager import EventLogManager
+from exo.shared.keypair import Keypair, get_node_id_keypair
from exo.shared.types.common import CommandId, NodeId
from exo.shared.types.events import (
Event,
@@ -34,7 +35,6 @@ from exo.shared.types.events.commands import (
from exo.shared.types.state import State
from exo.shared.types.tasks import ChatCompletionTask, TaskId, TaskStatus, TaskType
from exo.shared.types.worker.instances import Instance
-from exo.shared.utils import Keypair, get_node_id_keypair
class Master:
@@ -263,8 +263,8 @@ async def async_main():
command_buffer,
global_events,
worker_events,
- Path(os.environ["GO_BUILD_DIR"]) / "forwarder",
- logger,
+ forwarder_binary_path=Path(os.environ["GO_BUILD_DIR"]) / "forwarder",
+ logger=logger,
)
await master.run()
diff --git a/src/exo/master/tests/test_forwarder_supervisor.py b/src/exo/master/tests/test_forwarder_supervisor.py
index 00829696..1ac45bbd 100644
--- a/src/exo/master/tests/test_forwarder_supervisor.py
+++ b/src/exo/master/tests/test_forwarder_supervisor.py
@@ -393,6 +393,6 @@ class TestElectionCallbacks:
callbacks = ElectionCallbacks(mock_supervisor, test_logger)
await callbacks.on_became_replica()
- mock_supervisor.notify_role_change.assert_called_once_with(
+ mock_supervisor.notify_role_change.assert_called_once_with( # type: ignore
ForwarderRole.REPLICA
- ) # type: ignore
+ )
diff --git a/src/exo/master/tests/test_master.py b/src/exo/master/tests/test_master.py
index 293e454d..5e63ce52 100644
--- a/src/exo/master/tests/test_master.py
+++ b/src/exo/master/tests/test_master.py
@@ -10,6 +10,7 @@ from exo.master.main import Master
from exo.shared.db.sqlite.config import EventLogConfig
from exo.shared.db.sqlite.connector import AsyncSQLiteEventStorage
from exo.shared.db.sqlite.event_log_manager import EventLogManager
+from exo.shared.keypair import Keypair
from exo.shared.types.api import ChatCompletionMessage, ChatCompletionTaskParams
from exo.shared.types.common import NodeId
from exo.shared.types.events import Event, EventFromEventLog, Heartbeat, TaskCreated
@@ -31,10 +32,13 @@ from exo.shared.types.profiling import (
SystemPerformanceProfile,
)
from exo.shared.types.tasks import ChatCompletionTask, TaskStatus, TaskType
-from exo.shared.types.worker.common import InstanceId
-from exo.shared.types.worker.instances import Instance, InstanceStatus, ShardAssignments
+from exo.shared.types.worker.instances import (
+ Instance,
+ InstanceId,
+ InstanceStatus,
+ ShardAssignments,
+)
from exo.shared.types.worker.shards import PartitionStrategy, PipelineShardMetadata
-from exo.shared.utils import Keypair
def _create_forwarder_dummy_binary() -> Path:
diff --git a/src/exo/shared/constants.py b/src/exo/shared/constants.py
index fe1393c3..eb7b7ba9 100644
--- a/src/exo/shared/constants.py
+++ b/src/exo/shared/constants.py
@@ -16,6 +16,8 @@ EXO_NODE_ID_KEYPAIR = EXO_HOME / "node_id.keypair"
EXO_WORKER_KEYRING_FILE = EXO_HOME / "worker_keyring"
EXO_MASTER_KEYRING_FILE = EXO_HOME / "master_keyring"
+EXO_IPC_DIR = EXO_HOME / "ipc"
+
# libp2p topics for event forwarding
LIBP2P_WORKER_EVENTS_TOPIC = "worker_events"
LIBP2P_GLOBAL_EVENTS_TOPIC = "global_events"
diff --git a/src/exo/shared/db/sqlite/event_log_manager.py b/src/exo/shared/db/sqlite/event_log_manager.py
index 9a1aa1d9..571d6c8c 100644
--- a/src/exo/shared/db/sqlite/event_log_manager.py
+++ b/src/exo/shared/db/sqlite/event_log_manager.py
@@ -7,6 +7,7 @@ from sqlalchemy.exc import OperationalError
from exo.shared.constants import EXO_HOME
from exo.shared.db.sqlite.config import EventLogConfig, EventLogType
from exo.shared.db.sqlite.connector import AsyncSQLiteEventStorage
+from exo.shared.utils.fs import ensure_directory_exists
class EventLogManager:
@@ -25,7 +26,7 @@ class EventLogManager:
self._connectors: Dict[EventLogType, AsyncSQLiteEventStorage] = {}
# Ensure base directory exists
- EXO_HOME.mkdir(parents=True, exist_ok=True)
+ ensure_directory_exists(EXO_HOME)
# TODO: This seems like it's a pattern to avoid an async __init__ function. But as we know, there's a better pattern for this - using a create() function, like in runner_supervisor.
async def initialize(self, max_retries: int = 3) -> None:
diff --git a/src/exo/shared/ipc/__init__.py b/src/exo/shared/ipc/__init__.py
new file mode 100644
index 00000000..c6f0a7bd
--- /dev/null
+++ b/src/exo/shared/ipc/__init__.py
@@ -0,0 +1,14 @@
+"""
+A set of IPC primitives intended for cross-language use.
+Includes things like file-locks, named-pipe duplexes, and so on.
+
+TODO: implement System V IPC primitives??
+ 1. semaphores w/ SEM_UNDO flag ???
+ 2. Message Queues => as a replacement for pipe duplexes???
+ see: https://www.softprayog.in/programming/system-v-semaphores
+ https://tldp.org/LDP/lpg/node21.html
+ https://tldp.org/LDP/tlk/ipc/ipc.html
+ https://docs.oracle.com/cd/E19683-01/816-5042/auto32/index.html
+ https://www.softprayog.in/programming/posix-semaphores
+
+"""
diff --git a/src/exo/shared/ipc/file_mutex/__init__.py b/src/exo/shared/ipc/file_mutex/__init__.py
new file mode 100644
index 00000000..f8465963
--- /dev/null
+++ b/src/exo/shared/ipc/file_mutex/__init__.py
@@ -0,0 +1,4 @@
+"""
+A file-lock based IPC mutex primitives.
+
+"""
diff --git a/src/exo/shared/ipc/file_mutex/flock_mutex.py b/src/exo/shared/ipc/file_mutex/flock_mutex.py
new file mode 100644
index 00000000..fda65d60
--- /dev/null
+++ b/src/exo/shared/ipc/file_mutex/flock_mutex.py
@@ -0,0 +1,147 @@
+"""
+File-based mutex primitive implemented using UNIX-based `flock` syscall.
+
+"""
+
+import contextlib
+import errno
+import fcntl
+import os
+import stat
+import time
+from enum import Enum
+from typing import Optional
+
+from exo.shared.utils.fs import StrPath, ensure_parent_directory_exists
+
+# open in read-write mode, creates file if it doesn't exist already,
+# closes this file descriptor in any children processes (prevents FD leaking),
+# truncates this file on opening (lock-files shouldn't hold content FOR NOW!!!)
+# SEE: https://man7.org/linux/man-pages/man2/openat.2.html
+OPEN_FLAGS = os.O_RDWR | os.O_CREAT | os.O_CLOEXEC | os.O_TRUNC
+
+# 0x644 mode flags -> user has read-write permissions, others have read permission only
+# SEE: https://man7.org/linux/man-pages/man2/openat.2.html
+MODE_FLAGS = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+
+# default poll-interval for spin-blocking lock
+POLL_INTERVAL = 0.05
+
+
+class LockType(Enum):
+ READ = fcntl.LOCK_SH
+ WRITE = fcntl.LOCK_EX
+
+
+class AcquireMode(Enum):
+ OS_BLOCKING = 0
+ SPIN_BLOCKING = 1
+ NON_BLOCKING = 2
+
+
+class FlockMutex:
+ def __init__(self, file_path: StrPath):
+ self._file_path = file_path
+ self._fd: Optional[int] = None
+ self.lock_held: Optional[LockType] = None
+
+ def _open_fd(self):
+ assert self._fd is None
+ ensure_parent_directory_exists(self._file_path)
+
+ # open file & TRY to change permissions to `MODE_FLAGS` flags
+ self._fd = os.open(self._file_path, OPEN_FLAGS, MODE_FLAGS)
+ with contextlib.suppress(
+ PermissionError
+ ): # This locked is not owned by this UID
+ os.chmod(self._fd, MODE_FLAGS)
+
+ def _close_fd(self):
+ assert self._fd is not None
+ os.close(self._fd)
+ self._fd = None
+
+ def _acquire(self, lock_type: LockType, blocking: bool) -> bool:
+ assert (self._fd is not None) and (self.lock_held is None)
+
+ # create flags for acquiring lock
+ flags = lock_type.value
+ if not blocking:
+ flags |= fcntl.LOCK_NB
+
+ # continually try to acquire lock (since it may fail due to interrupts)
+ while True:
+ try:
+ fcntl.flock(self._fd, flags)
+ break
+ except OSError as e:
+ if e.errno == errno.EINTR: # call interrupted by signal -> try again
+ continue
+ elif (
+ e.errno == errno.EWOULDBLOCK
+ ): # file is locked & non-blocking is enabled -> return false to indicate
+ return False
+
+ # unhandleable errors -> close FD & raise
+ self._close_fd()
+ if e.errno == errno.ENOSYS: # NotImplemented error
+ raise NotImplementedError(
+ "This system doesn't support flock"
+ ) from e
+ else:
+ raise
+
+ # set lock-type held
+ self.lock_held = lock_type
+ return True
+
+ def _release(self):
+ assert (self._fd is not None) and (self.lock_held is not None)
+
+ # continually try to release lock (since it may fail due to interrupts)
+ while True:
+ try:
+ fcntl.flock(self._fd, fcntl.LOCK_UN)
+ break
+ except OSError as e:
+ if e.errno == errno.EINTR: # call interrupted by signal -> try again
+ continue
+
+ # unhandleable errors -> close FD & raise
+ self._close_fd()
+ if e.errno == errno.ENOSYS: # NotImplemented error
+ raise NotImplementedError(
+ "This system doesn't support flock"
+ ) from e
+ else:
+ raise
+
+ self.lock_held = None
+
+ def acquire(
+ self,
+ lock_type: LockType = LockType.WRITE,
+ acquire_mode: AcquireMode = AcquireMode.SPIN_BLOCKING,
+ ) -> bool:
+ if self._fd is None:
+ self._open_fd()
+
+ # OS-blocking & non-blocking is direct passthrough to private function
+ match acquire_mode:
+ case AcquireMode.OS_BLOCKING:
+ return self._acquire(lock_type, blocking=True)
+ case AcquireMode.NON_BLOCKING:
+ return self._acquire(lock_type, blocking=False)
+ case _:
+ pass
+
+ # spin-blocking works by trying to acquire the lock in non-blocking mode, and retrying until success
+ while True:
+ locked = self._acquire(lock_type, blocking=False)
+ if locked:
+ return True
+ time.sleep(POLL_INTERVAL)
+
+ def release(self):
+ self._release()
+ self._close_fd()
diff --git a/src/exo/shared/ipc/pipe_duplex.py b/src/exo/shared/ipc/pipe_duplex.py
new file mode 100644
index 00000000..3ba5a98e
--- /dev/null
+++ b/src/exo/shared/ipc/pipe_duplex.py
@@ -0,0 +1,415 @@
+"""
+SEE:
+ - https://pubs.opengroup.org/onlinepubs/007904875/functions/open.html
+ - https://man7.org/linux/man-pages/man2/openat.2.html
+ - https://man7.org/linux/man-pages/man3/mkfifo.3.html
+ - https://man7.org/linux/man-pages/man7/pipe.7.html
+
+TODO: add locking on reader/writer ends to prevent multiwriters??
+TODO: use signal bytes to ensure proper packet consistency
+ +stretch: implement packet IDs, retries, dual-stream confirmations, RPCs & so on
+
+TODO: for more hardening -> check if any of the syscalls used return signal interrupt errors (like in the locking case)
+ and interrupt on that happening -> this may not be an issue PER SE but might potentially create insanely bizzare bugs
+ if it happens that this behavior DOES occasionally happen for no apparent reason
+
+TODO: maybe consider padding all messages with 0s on both ends ?? so as to prevent ANY ambiguous boundaries ever!!
+"""
+
+import errno
+import logging
+import multiprocessing
+import os
+import queue
+import stat
+import threading
+import time
+from enum import Enum
+from multiprocessing.queues import Queue as MQueueT
+from multiprocessing.synchronize import Event as MEventT
+from threading import Event as TEventT
+from typing import Callable
+
+from cobs import cobs # pyright: ignore[reportMissingTypeStubs]
+from pytest import LogCaptureFixture
+
+from exo.shared.utils.fs import (
+ StrPath,
+ delete_if_exists,
+ ensure_parent_directory_exists,
+)
+
+OPEN_READER_FLAGS = os.O_RDONLY | os.O_NONBLOCK
+OPEN_WRITER_FLAGS = os.O_WRONLY | os.O_NONBLOCK
+
+# 0x644 mode flags -> user has read-write permissions, others have read permission only
+MODE_FLAGS = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+
+POLL_INTERVAL = 0.05 # TODO: maybe parametrize this in classes??
+PIPE_BUF = 4096 # size of atomic writes on (most) UNIX pipes
+
+
+class SignalMessage(Enum):
+ """
+ Signal messages range from 1 to 255 & indicate control flow for the bytestream of the pipe.
+
+ """
+
+ DISCARD_PREVIOUS = b"\x01"
+
+
+class PipeDuplex:
+ """
+ Creates a named-pipe communication duplex. The reader end is responsible for creating the pipe.
+
+ The layers are:
+ 1. Raw binary data over pipes
+ 2. Variable-length binary packets with COBS
+ 3. JSON-like values with Message Pack
+ """
+
+ def __init__(
+ self,
+ in_pipe: StrPath,
+ out_pipe: StrPath,
+ in_callback: Callable[[bytes], None],
+ ):
+ assert in_pipe != out_pipe # they must be different files
+
+ # pipes should only ever be created, and only by the reader (one-way operations)
+ _ensure_fifo_exists(in_pipe) # ensures reader pipe exists
+
+ # create readonly properties (useful for inspection)
+ self._in_pipe = in_pipe
+ self._out_pipe = out_pipe
+
+ # init synchronisation variables
+ self._mkill = multiprocessing.Event()
+ self._tkill = threading.Event()
+ in_mq: MQueueT[bytes] = multiprocessing.Queue()
+ self._out_mq: MQueueT[bytes] = multiprocessing.Queue()
+ in_mstarted = multiprocessing.Event()
+
+ # process for reading in binary messages from pipe
+ self._p_in = multiprocessing.Process(
+ target=_pipe_buffer_reader,
+ args=(in_pipe, in_mq, in_mstarted, self._mkill),
+ daemon=True,
+ )
+ self._p_in.start()
+
+ # thread for pulling down binary messages from message queue & calling the callback
+ self._t_in = threading.Thread(
+ target=_binary_object_dispatcher,
+ args=(in_mq, in_callback, self._tkill),
+ daemon=True,
+ )
+ self._t_in.start()
+
+ # process to write binary messages to pipe
+ out_mstarted = multiprocessing.Event()
+ self._p_out = multiprocessing.Process(
+ target=_pipe_buffer_writer,
+ args=(out_pipe, self._out_mq, out_mstarted, self._mkill),
+ daemon=True,
+ )
+ self._p_out.start()
+
+ # wait for processes to start properly
+ in_mstarted.wait()
+ out_mstarted.wait()
+
+ def __del__(self):
+ # signal to these processes to die (if they haven't already)
+ self._mkill.set()
+ self._tkill.set()
+
+ def send_message(self, msg: bytes):
+ self._out_mq.put_nowait(msg)
+
+ @property
+ def in_pipe(self):
+ return self._in_pipe
+
+ @property
+ def out_pipe(self):
+ return self._out_pipe
+
+
+def _ensure_fifo_exists(path: StrPath):
+ # try to make a file if one doesn't exist already
+ ensure_parent_directory_exists(path)
+ try:
+ os.mkfifo(path, mode=MODE_FLAGS)
+ except OSError as e:
+ # misc error, do not handle
+ if e.errno != errno.EEXIST:
+ raise
+
+ # ensure the file exists is FIFO
+ st = os.stat(path)
+ if stat.S_ISFIFO(st.st_mode):
+ return
+
+ # this file is not FIFO
+ raise FileExistsError(f"The file '{path}' isn't a FIFO") from e
+
+
+def _pipe_buffer_reader(
+ path: StrPath, mq: MQueueT[bytes], started: MEventT, kill: MEventT
+):
+ # TODO: right now the `kill` control flow is somewhat haphazard -> ensure every loop-y or blocking part always
+ # checks for kill.is_set() and returns/cleans up early if so
+
+ # open reader in nonblocking mode -> should not fail & immediately open;
+ # this marks when the writer process has "started"
+ fd = os.open(path, OPEN_READER_FLAGS)
+ started.set()
+ print("(reader):", "started")
+
+ # continually pull from the pipe and interpret messages as such:
+ # - all messages are separated/framed by NULL bytes (zero)
+ # - messages with >=2 bytes are COBS-encoded messages, because
+ # the smallest COBS-encoded message is 2 bytes
+ # - 1-byte messages are therefore to be treated as control signals
+ #
+ # TODO: right now i just need to get this to work, but the scheme is fundamentally
+ # extensible for robustness, e.g. signal-bytes can be used to drive state-machines
+ # for ensuring message atomicity/transmission
+ # e.g. we can use single-bytes to discriminate COBS values to say "this is length of upcoming message"
+ # vs. this is the actual content of the message, and so on
+ # .
+ # BUT for now we can just use signal (0xff 0x00) to mean "discard previous message" or similar...
+ # .
+ # BUT in THEORY we very well could have something like
+ # (0x10 0x00)[header signal] + (...)[header data like length & so on]
+ # + (0x20 0x00)[body signal] + (...)[body data]
+ # + (0x30 0x00)[checksum signal] + (...)[checksum data]
+ # And requests to re-send messages that were lost, and so on, like this is a fully 2-layer duplex
+ # communication so we could turn this into a VERY powerful thing some time in the future, like
+ # a whole-ass reimplementation of TCP/PIPES lmaooooo
+ buffer = bytearray()
+ while not kill.is_set():
+ try:
+ # read available data (and try again if nothing)
+ try:
+ data = os.read(fd, PIPE_BUF)
+ if data == b"":
+ time.sleep(POLL_INTERVAL)
+ continue
+ except OSError as e:
+ if e.errno != errno.EAGAIN:
+ raise
+
+ # if there is a writer connected & the buffer is empty, this would block
+ # so we must consume this error gracefully and try again
+ time.sleep(POLL_INTERVAL)
+ continue
+
+ # extend buffer with new data
+ buffer.extend(data)
+
+ # if there are no NULL bytes in the buffer, no new message has been formed
+ chunks = buffer.split(sep=b"\x00")
+ if len(chunks) == 1:
+ continue
+
+ # last chunk is always an unfinished message, so that becomes our new buffer;
+ # the rest should be decoded as either signals or COBS and put on queue
+ buffer = chunks.pop()
+ for chunk in chunks:
+ chunk = bytes(chunk)
+
+ # ignore empty messages (they mean nothing)
+ if chunk == b"":
+ continue
+
+ # interpret 1-byte messages as signals (they indicate control-flow on messages)
+ if len(chunk) == 1:
+ print("(reader):", f"gotten control signal: {chunk[0]}")
+ continue # TODO: right now they should be ignored, since I'm not sure what I want them to do
+
+ # interpret >=2 byte messages as COBS-encoded data (decode them)
+ decoded = cobs.decode(chunk) # pyright: ignore[reportUnknownMemberType]
+ mq.put(decoded)
+ except BaseException as e:
+ # perform cleanup & log before re-raising
+ os.close(fd)
+ logging.error(msg=f"Error when reading from named pipe at '{path}': {e}")
+ raise
+ os.close(fd)
+
+
+def _binary_object_dispatcher(
+ mq: MQueueT[bytes], callback: Callable[[bytes], None], kill: TEventT
+):
+ while not kill.is_set():
+ # try to get with timeout (to allow to read the kill-flag)
+ try:
+ message = mq.get(block=True, timeout=POLL_INTERVAL)
+ except queue.Empty:
+ continue
+
+ # dispatch binary object with callback
+ callback(message)
+
+
+def _pipe_buffer_writer(
+ path: StrPath, mq: MQueueT[bytes], started: MEventT, kill: MEventT
+):
+ # TODO: right now the `kill` control flow is somewhat haphazard -> ensure every loop-y or blocking part always
+ # checks for kill.is_set() and returns/cleans up early if so
+
+ # for now, started events for writer are rather vacuous: TODO: remove or make more usefull??
+ started.set()
+ print("(writer):", "started")
+
+ # continually attempt to open FIFO for reading in nonblocking mode -> will error that:
+ # - ENOENT[2] No such file or directory: until a reader creates FIFO
+ # - ENXIO[6] No such device or address: until a reader opens FIFO
+ fd = None
+ while not kill.is_set():
+ try:
+ fd = os.open(path, os.O_WRONLY | os.O_NONBLOCK)
+
+ # ensure the file exists is FIFO
+ st = os.fstat(fd)
+ print("mode:", st.st_mode & 0o170000)
+ if stat.S_ISFIFO(st.st_mode):
+ break
+
+ # cleanup on error
+ os.close(fd)
+ raise FileExistsError(f"The file '{path}' isn't a FIFO")
+ except FileExistsError:
+ raise # propagate error
+ except OSError as e:
+ # misc error, do not handle
+ if not (e.errno == errno.ENOENT or e.errno == errno.ENXIO):
+ raise
+
+ # try again if waiting for FIFO creation or reader-end opening
+ time.sleep(POLL_INTERVAL)
+ continue
+ assert fd is not None
+
+ while not kill.is_set():
+ try:
+ # try to get with timeout (to allow to read the kill-flag)
+ try:
+ data = mq.get(block=True, timeout=POLL_INTERVAL)
+ except queue.Empty:
+ continue
+
+ # write all data (by continually re-trying until it is done)
+ _write_data(fd, data)
+ except BaseException as e:
+ # perform cleanup & log before re-raising
+ os.close(fd)
+ logging.error(msg=f"Error when writing to named pipe at '{path}': {e}")
+ raise
+
+ os.close(fd)
+
+
+def _write_data(fd: int, buf: bytes):
+ # COBS-encode the data & append NULL-byte to signify end-of-frame
+ buf = cobs.encode(buf) + b"\x00" # pyright: ignore[reportUnknownMemberType]
+ total = len(buf)
+ sent = 0
+
+ # begin transmission progress
+ while sent < total:
+ try:
+ # Write remaining bytes to the pipe
+ written = os.write(fd, buf[sent:])
+ sent += written
+ except OSError as e:
+ # non-blocking pipe is full, wait a bit and retry
+ if e.errno == errno.EAGAIN:
+ time.sleep(POLL_INTERVAL)
+ continue
+
+ # reader disconnected -> handle failure-recovery by doing:
+ # 1. signal DISCARD_PREVIOUS to any reader
+ # 2. re-setting the progress & trying again
+ if e.errno == errno.EPIPE:
+ _write_signal(fd, SignalMessage.DISCARD_PREVIOUS)
+ sent = 0
+ continue
+
+ raise # misc error, do not handle
+
+
+def _write_signal(fd: int, signal: SignalMessage):
+ signal_message_length = 2
+
+ # Turn signal-byte into message by terminating with NULL-byte
+ buf = signal.value + b"\x00"
+ assert len(buf) == signal_message_length
+
+ # attempt to write until successful
+ while True:
+ try:
+ # small writes (e.g. 2 bytes) should be atomic as per Pipe semantics,
+ # meaning IF SUCCESSFUL: the number of bytes written MUST be exactly 2
+ written = os.write(fd, buf)
+ assert written == signal_message_length
+ break
+ except OSError as e:
+ # wait a bit and retry if:
+ # - non-blocking pipe is full
+ # - the pipe is broken because of reader disconnection
+ if e.errno == errno.EAGAIN or e.errno == errno.EPIPE:
+ time.sleep(POLL_INTERVAL)
+ continue
+
+ raise # misc error, do not handle
+
+
+def _test_one_two_three():
+ one_path = "/tmp/one.pipe"
+ two_path = "/tmp/two.pipe"
+ delete_if_exists(one_path)
+ delete_if_exists(two_path)
+
+ owner = PipeDuplex(
+ in_pipe=one_path,
+ out_pipe=two_path,
+ in_callback=lambda x: print(f"wow, owner got: [{len(x)}]{x}"),
+ )
+
+ guest = PipeDuplex(
+ in_pipe=two_path,
+ out_pipe=one_path,
+ in_callback=lambda x: print(f"wow, guest1 got: [{len(x)}]{x}"),
+ )
+
+ owner.send_message(bytes(0 for _ in range(10)))
+
+ guest.send_message(bytes(0 for _ in range(200)))
+
+ time.sleep(1)
+
+ del guest
+ guest = PipeDuplex(
+ in_pipe=two_path,
+ out_pipe=one_path,
+ in_callback=lambda x: print(f"wow, guest2 got: [{len(x)}]{x}"),
+ )
+
+ guest.send_message(bytes(0 for _ in range(21)))
+
+ owner.send_message(bytes(0 for _ in range(12)))
+
+ time.sleep(1)
+
+ delete_if_exists(one_path)
+ delete_if_exists(two_path)
+
+
+def test_running_pipe_duplex(caplog: LogCaptureFixture):
+ caplog.set_level(logging.INFO)
+
+ _test_one_two_three()
+ time.sleep(1)
diff --git a/src/exo/shared/utils.py b/src/exo/shared/keypair.py
similarity index 97%
rename from src/exo/shared/utils.py
rename to src/exo/shared/keypair.py
index a819e7fb..a78c2cb4 100644
--- a/src/exo/shared/utils.py
+++ b/src/exo/shared/keypair.py
@@ -4,7 +4,7 @@ import hashlib
import logging
import os
from pathlib import Path
-from typing import Any, Type, final
+from typing import final
import base58
from cryptography.hazmat.primitives import serialization
@@ -216,12 +216,6 @@ class Keypair:
return self._public_key
-def ensure_type[T](obj: Any, expected_type: Type[T]) -> T: # type: ignore
- if not isinstance(obj, expected_type):
- raise TypeError(f"Expected {expected_type}, got {type(obj)}") # type: ignore
- return obj
-
-
def get_node_id_keypair(
path: str | bytes | os.PathLike[str] | os.PathLike[bytes] = EXO_NODE_ID_KEYPAIR,
) -> Keypair:
diff --git a/src/exo/shared/tests/test_flock_mutex.py b/src/exo/shared/tests/test_flock_mutex.py
new file mode 100644
index 00000000..42d68753
--- /dev/null
+++ b/src/exo/shared/tests/test_flock_mutex.py
@@ -0,0 +1,48 @@
+import pytest
+
+from exo.shared.ipc.file_mutex.flock_mutex import FlockMutex, LockType
+from exo.shared.utils.fs import delete_if_exists, make_temp_path
+
+
+def test_lock_held():
+ path = make_temp_path("testing_flock.lock")
+ lock = FlockMutex(path)
+
+ assert lock.lock_held is None
+
+ assert lock.acquire(lock_type=LockType.WRITE)
+ assert lock.lock_held == LockType.WRITE
+ lock.release()
+
+ assert lock.lock_held is None
+
+ assert lock.acquire(lock_type=LockType.READ)
+ assert lock.lock_held == LockType.READ
+ lock.release()
+
+ assert lock.lock_held is None
+
+ delete_if_exists(path)
+
+
+def test_no_reentrant_lock():
+ path = make_temp_path("testing_flock.lock")
+ lock = FlockMutex(path)
+
+ # no write-lock reentrancy
+ lock.acquire(lock_type=LockType.WRITE)
+ with pytest.raises(AssertionError):
+ lock.acquire(lock_type=LockType.WRITE)
+ with pytest.raises(AssertionError):
+ lock.acquire(lock_type=LockType.READ)
+ lock.release()
+
+ # no read-lock reentrancy
+ lock.acquire(lock_type=LockType.READ)
+ with pytest.raises(AssertionError):
+ lock.acquire(lock_type=LockType.WRITE)
+ with pytest.raises(AssertionError):
+ lock.acquire(lock_type=LockType.READ)
+ lock.release()
+
+ delete_if_exists(path)
diff --git a/src/exo/shared/tests/test_node_id_persistence.py b/src/exo/shared/tests/test_node_id_persistence.py
index 552311e7..46a81d55 100644
--- a/src/exo/shared/tests/test_node_id_persistence.py
+++ b/src/exo/shared/tests/test_node_id_persistence.py
@@ -14,7 +14,7 @@ from typing import Optional
from pytest import LogCaptureFixture
from exo.shared.constants import EXO_NODE_ID_KEYPAIR
-from exo.shared.utils import get_node_id_keypair
+from exo.shared.keypair import get_node_id_keypair
NUM_CONCURRENT_PROCS = 10
diff --git a/src/exo/shared/types/events/_events.py b/src/exo/shared/types/events/_events.py
index c59a2df1..dccb9f6f 100644
--- a/src/exo/shared/types/events/_events.py
+++ b/src/exo/shared/types/events/_events.py
@@ -297,8 +297,8 @@ def _check_event_type_consistency():
# grab type hints and extract the right values from it
cls_hints = get_type_hints(cls)
assert (
- "event_type" in cls_hints and get_origin(cls_hints["event_type"]) is Literal
- ), ( # pyright: ignore[reportAny]
+ "event_type" in cls_hints and get_origin(cls_hints["event_type"]) is Literal # type: ignore
+ ), (
f"{get_error_reporting_message()}",
f"The class {cls} is missing a {Literal}-annotated `event_type` field.",
)
diff --git a/src/exo/shared/utils/__init__.py b/src/exo/shared/utils/__init__.py
new file mode 100644
index 00000000..87131484
--- /dev/null
+++ b/src/exo/shared/utils/__init__.py
@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import Any, Type
+
+from exo.shared.utils.phantom import PhantomData
+
+
+def ensure_type[T](obj: Any, expected_type: Type[T]) -> T: # type: ignore
+ if not isinstance(obj, expected_type):
+ raise TypeError(f"Expected {expected_type}, got {type(obj)}") # type: ignore
+ return obj
+
+
+def todo[T](
+ msg: str = "This code has not been implemented yet.",
+ _phantom: PhantomData[T] = None,
+) -> T:
+ raise NotImplementedError(msg)
diff --git a/src/exo/shared/utils/fs.py b/src/exo/shared/utils/fs.py
new file mode 100644
index 00000000..a72a73ba
--- /dev/null
+++ b/src/exo/shared/utils/fs.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import contextlib
+import os
+import pathlib
+import tempfile
+from typing import LiteralString
+
+type StrPath = str | os.PathLike[str]
+type BytesPath = bytes | os.PathLike[bytes]
+type StrOrBytesPath = str | bytes | os.PathLike[str] | os.PathLike[bytes]
+
+
+def delete_if_exists(filename: StrOrBytesPath) -> None:
+ with contextlib.suppress(FileNotFoundError):
+ os.remove(filename)
+
+
+def ensure_parent_directory_exists(filename: StrPath) -> None:
+ """
+ Ensure the directory containing the file exists (create it if necessary).
+ """
+ pathlib.Path(filename).parent.mkdir(parents=True, exist_ok=True)
+
+
+def ensure_directory_exists(dirname: StrPath) -> None:
+ """
+ Ensure the directory exists (create it if necessary).
+ """
+ pathlib.Path(dirname).mkdir(parents=True, exist_ok=True)
+
+
+def make_temp_path(name: LiteralString) -> str:
+ return os.path.join(tempfile.mkdtemp(), name)
diff --git a/src/exo/shared/utils/phantom.py b/src/exo/shared/utils/phantom.py
new file mode 100644
index 00000000..7311ea6e
--- /dev/null
+++ b/src/exo/shared/utils/phantom.py
@@ -0,0 +1,14 @@
+from typing import Optional
+
+
+class _PhantomData[T]:
+ """
+ Internal machinery of the phantom data - it stores nothing.
+ """
+
+
+type PhantomData[T] = Optional[_PhantomData[T]]
+"""
+Allows you to use generics in functions without storing anything of that generic type.
+Just use `None` and you'll be fine
+"""
diff --git a/src/exo/shared/utils/pydantic_ext.py b/src/exo/shared/utils/pydantic_ext.py
new file mode 100644
index 00000000..e85591f7
--- /dev/null
+++ b/src/exo/shared/utils/pydantic_ext.py
@@ -0,0 +1,52 @@
+from pydantic import BaseModel
+from pydantic.alias_generators import to_camel
+
+
+class CamelCaseModel(BaseModel):
+ """
+ A model whose fields are aliased to camel-case from snake-case.
+ """
+
+ class Config:
+ alias_generator = to_camel
+ allow_population_by_field_name = True
+
+
+class Tagged[Tag: str, Content](
+ CamelCaseModel
+): # TODO: figure out how to make pydantic work with LiteralString
+ """
+ Utility for helping with serializing unions as adjacently tagged with Pydantic.
+
+ By default, Pydantic uses internally tagged union ser/de BUT to play nicely with
+ other cross-language ser/de tools, you need adjacently tagged unions, and Pydantic
+ doesn't support those out of the box.
+
+ SEE: https://serde.rs/enum-representations.html#adjacently-tagged
+
+ Example usage:
+ ```python
+ TaggedUnion = Annotated[Union[
+ Tagged[Literal["Foo"], Foo],
+ Tagged[Literal["Bar"], Bar]
+ ], Field(discriminator="t")]
+
+ Parser: TypeAdapter[TaggedUnion] = TypeAdapter(TaggedUnion)
+
+ def validate_python(v: any) -> Foo | Bar:
+ v = Parser.validate_python(v)
+ match v.t:
+ case "Foo": return v.c
+ case "Bar": return v.c
+ ```
+ """
+
+ t: Tag
+ """
+ The tag corresponding to the type of the object in the union.
+ """
+
+ c: Content
+ """
+ The actual content of the object of that type.
+ """
diff --git a/src/exo/shared/utils/reactive.py b/src/exo/shared/utils/reactive.py
new file mode 100644
index 00000000..14c021d2
--- /dev/null
+++ b/src/exo/shared/utils/reactive.py
@@ -0,0 +1,32 @@
+"""
+Utilities for reactive variables
+
+"""
+
+from typing import Protocol
+
+
+class OnChange[T](Protocol):
+ def __call__(self, old_value: T, new_value: T) -> None: ...
+
+
+class Reactive[T]:
+ def __init__(self, initial_value: T, on_change: OnChange[T]):
+ self._value = initial_value
+ self._on_change = on_change
+
+ @property
+ def value(self):
+ return self._value
+
+ @value.setter
+ def value(self, new_value: T):
+ old_value = self._value
+ self._value = new_value
+
+ # don't notify when not changed
+ if old_value == new_value:
+ return
+
+ # notify of changes
+ self._on_change(old_value=old_value, new_value=new_value)
diff --git a/src/exo/worker/main.py b/src/exo/worker/main.py
index 621d0cc1..abd9af78 100644
--- a/src/exo/worker/main.py
+++ b/src/exo/worker/main.py
@@ -3,6 +3,7 @@ import logging
from exo.shared.apply import apply
from exo.shared.db.sqlite.event_log_manager import EventLogConfig, EventLogManager
+from exo.shared.keypair import Keypair, get_node_id_keypair
from exo.shared.types.common import NodeId
from exo.shared.types.events import (
NodePerformanceMeasured,
@@ -12,7 +13,6 @@ from exo.shared.types.worker.ops import (
ExecuteTaskOp,
RunnerOp,
)
-from exo.shared.utils import Keypair, get_node_id_keypair
from exo.worker.download.impl_shard_downloader import exo_shard_downloader
from exo.worker.plan import plan
from exo.worker.utils.profile import start_polling_node_metrics
diff --git a/src/exo/worker/runner/runner.py b/src/exo/worker/runner/runner.py
index 25e1a025..440dcdef 100644
--- a/src/exo/worker/runner/runner.py
+++ b/src/exo/worker/runner/runner.py
@@ -6,7 +6,7 @@ from functools import partial
from typing import Callable, cast
import mlx.core as mx
-import mlx.nn as nn
+import mlx.nn as nn # pyright: ignore [reportMissingTypeStubs]
from mlx_lm.generate import stream_generate # type: ignore
from mlx_lm.tokenizer_utils import TokenizerWrapper
diff --git a/src/exo/worker/tests/test_spinup_timeout.py b/src/exo/worker/tests/test_spinup_timeout.py
index 8649fef9..501ca649 100644
--- a/src/exo/worker/tests/test_spinup_timeout.py
+++ b/src/exo/worker/tests/test_spinup_timeout.py
@@ -37,9 +37,9 @@ async def test_runner_up_op_timeout(
# _execute_runner_up_op should throw a TimeoutError with a short timeout
events: list[Event] = []
- async for event in worker._execute_runner_up_op(
+ async for event in worker._execute_runner_up_op( # type: ignore[misc]
runner_up_op, initialize_timeout=0.2
- ): # type: ignore[misc]
+ ):
events.append(event)
assert isinstance(events[-1], RunnerStatusUpdated)
diff --git a/uv.lock b/uv.lock
index f8fb5d9f..bb4af869 100644
--- a/uv.lock
+++ b/uv.lock
@@ -130,15 +130,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4a/45/ec96b29162a402fc4c1c5512d114d7b3787b9d1c2ec241d9568b4816ee23/base58-2.1.1-py3-none-any.whl", hash = "sha256:11a36f4d3ce51dfc1043f3218591ac4eb1ceb172919cebe05b52a5bcc8d245c2", size = 5621, upload-time = "2021-10-30T22:12:16.658Z" },
]
-[[package]]
-name = "braq"
-version = "0.0.12"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/54/3b/1b918c408e11ca33f9b9dcecc8e08eac7762887dd42b584f0efb6fe26c55/braq-0.0.12.tar.gz", hash = "sha256:51dae51b863cbba2cd37da163df06b7dc5124904d2c26b92bda54c1bde66d74b", size = 15272, upload-time = "2024-12-10T20:48:53.856Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/f3/53/ed5082619966b1d15b5c039ac722ba99956d92d4b08a9bd5eb4c3535cc1f/braq-0.0.12-py3-none-any.whl", hash = "sha256:41b7bdd0d004faef693751615fbb11c53ac0b886c772b83aea61ea6dc2f6e518", size = 26392, upload-time = "2024-12-10T20:48:50.813Z" },
-]
-
[[package]]
name = "certifi"
version = "2025.8.3"
@@ -204,6 +195,12 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" },
]
+[[package]]
+name = "cobs"
+version = "1.2.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/34/ef/ea149311227a4fc3160cc885fce06da7c7d76782a308ef070b8065c69953/cobs-1.2.2.tar.gz", hash = "sha256:dbdd5e32111d72786f83d0c269215dcd6ac629b1ac1962c6878221f3b2ca98da", size = 14582, upload-time = "2025-07-20T01:08:35.434Z" }
+
[[package]]
name = "cryptography"
version = "45.0.6"
@@ -253,6 +250,7 @@ dependencies = [
{ name = "aiohttp", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "aiosqlite", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "base58", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "cobs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "cryptography", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "fastapi", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -294,6 +292,7 @@ requires-dist = [
{ name = "aiohttp", specifier = ">=3.12.14" },
{ name = "aiosqlite", specifier = ">=0.21.0" },
{ name = "base58", specifier = ">=2.1.1" },
+ { name = "cobs", specifier = ">=1.2.2" },
{ name = "cryptography", specifier = ">=45.0.5" },
{ name = "fastapi", specifier = ">=0.116.1" },
{ name = "filelock", specifier = ">=3.18.0" },
@@ -331,14 +330,14 @@ name = "exo-scripts"
version = "0.1.0"
source = { editable = "scripts" }
dependencies = [
+ { name = "exo", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
- { name = "shared", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
[package.metadata]
requires-dist = [
+ { name = "exo", editable = "." },
{ name = "huggingface-hub", specifier = ">=0.33.4" },
- { name = "shared" },
]
[[package]]
@@ -566,19 +565,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" },
]
-[[package]]
-name = "kvf"
-version = "0.0.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "braq", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
- { name = "paradict", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9c/f8/e1826c156d4f97cf4662a6110cbbcfd91b5e5570c8a88bf0a8270718621e/kvf-0.0.3.tar.gz", hash = "sha256:f4885b1bbe66c8c20fdabe5cedeb3c0e5d12a54ac495f9e5fcf6fed0e0c51b73", size = 4938, upload-time = "2024-12-10T20:49:13.171Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/a8/db/4a8d3b1fef45cabcadf36f9a2231b2cde3dddd3a58ab1723119c7fbce34f/kvf-0.0.3-py3-none-any.whl", hash = "sha256:9d666e51cae512e3f95c55b77524e34d0095b278c81f96f7bbc7d37b5bd545c6", size = 4716, upload-time = "2024-12-10T20:49:11.815Z" },
-]
-
[[package]]
name = "markdown-it-py"
version = "4.0.0"
@@ -764,15 +750,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },
]
-[[package]]
-name = "paradict"
-version = "0.0.16"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/40/83/8cf8d94be55ab9ea783e1f8ece06059cd986bb482ad69f7be549839b9e07/paradict-0.0.16.tar.gz", hash = "sha256:d909d122bf47028a45334eb2280d1e1bcb401fda89986af42c39fd2fadf9de4d", size = 61471, upload-time = "2024-12-10T21:23:49.007Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/1d/f9/a9807d307ba1837bb8799e1337f41edcdbb92ef6090668dc50f483a168bf/paradict-0.0.16-py3-none-any.whl", hash = "sha256:28df79f0dc0e68c8f8a3e9b7c75e67a85305ef7298653fc7a369a1bf4f58cb20", size = 61735, upload-time = "2024-12-10T21:23:45.408Z" },
-]
-
[[package]]
name = "pathlib"
version = "1.0.1"
@@ -1072,19 +1049,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ad/fe/cad1d9762868c7c5dc70c8620074df28ebb1a8e4c17d4c0cb031889c457e/safetensors-0.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d944cea65fad0ead848b6ec2c37cc0b197194bec228f8020054742190e9312ac", size = 655957, upload-time = "2025-08-08T13:13:57.029Z" },
]
-[[package]]
-name = "shared"
-version = "0.0.32"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "kvf", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
- { name = "paradict", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/3f/39/f39c2560ac971efbf437f7ffa1d82a12fa77f50b0127e6e5ec5cc8d377df/shared-0.0.32.tar.gz", hash = "sha256:7308adc95c0dab14d0c99635cd8049d1f004cc7fef7396d3fe47323c34ec58c6", size = 7793, upload-time = "2024-12-10T20:49:22.469Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/f9/03/da58e40386d8ebcdfa3617070a95ca1deb5a5e6aa3d4e15ea2045173d5ac/shared-0.0.32-py3-none-any.whl", hash = "sha256:f17962c0f0fe6a23015accc7cac029e1c24c4b14578094e1f7033a7a7ef16140", size = 29304, upload-time = "2024-12-10T20:49:19.763Z" },
-]
-
[[package]]
name = "sniffio"
version = "1.3.1"
diff --git a/worker/pyproject.toml b/worker/pyproject.toml
deleted file mode 100644
index dca88c33..00000000
--- a/worker/pyproject.toml
+++ /dev/null
@@ -1,18 +0,0 @@
-[project]
-name = "worker"
-version = "0.1.0"
-description = "Worker for the Exo project"
-readme = "README.md"
-requires-python = ">=3.13"
-dependencies = [
- "shared",
- "huggingface_hub>=0.33.4",
- "mlx>=0.26.3",
- "mlx-lm @ https://github.com/ml-explore/mlx-lm.git",
- "psutil>=7.0.0",
- "transformers>=4.55.0",
-]
-
-[build-system]
-requires = ["uv_build>=0.8.9,<0.9.0"]
-build-backend = "uv_build"
← 5bfc99b4 add EXO logo to dashboard
·
back to Exo
·
feat: single entrypoint and logging rework 5efe5562 →