diff --git a/Cargo.lock b/Cargo.lock index 079f035..a7738d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,8 +8,10 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "flate2", "futures-util", "indicatif", + "reqwest", "serde", "serde_json", "sqlx", @@ -97,6 +99,19 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "async-compression" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b37fc50485c4f3f736a4fb14199f6d5f5ba008d7f28fe710306c92780f004c07" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "atoi" version = "2.0.0" @@ -106,6 +121,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.4.0" @@ -175,6 +196,15 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cc" +version = "1.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -255,6 +285,22 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -279,6 +325,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -359,12 +414,31 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -387,6 +461,22 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "flate2" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -398,12 +488,33 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -515,7 +626,19 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -524,6 +647,25 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "h2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.3" @@ -583,6 +725,124 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a01595e11bdcec50946522c32dde3fc6914743000a68b93000965f2f02406d" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + [[package]] name = "icu_collections" version = "2.0.0" @@ -719,6 +979,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -772,6 +1042,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + [[package]] name = "litemap" version = "0.8.0" @@ -810,6 +1086,12 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "miniz_oxide" version = "0.8.8" @@ -826,10 +1108,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -908,6 +1207,50 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" @@ -1039,6 +1382,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "rand" version = "0.8.5" @@ -1066,7 +1415,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.16", ] [[package]] @@ -1078,6 +1427,66 @@ dependencies = [ "bitflags", ] +[[package]] +name = "reqwest" +version = "0.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" +dependencies = [ + "async-compression", + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rsa" version = "0.9.8" @@ -1104,18 +1513,102 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.219" @@ -1200,6 +1693,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.5" @@ -1500,6 +1999,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1511,6 +2019,40 @@ dependencies = [ "syn", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "thiserror" version = "2.0.12" @@ -1595,6 +2137,26 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -1606,6 +2168,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.22" @@ -1647,6 +2222,51 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.41" @@ -1705,6 +2325,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.18.0" @@ -1744,6 +2370,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.4" @@ -1791,12 +2423,30 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -1811,6 +2461,7 @@ checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] @@ -1828,6 +2479,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -1860,6 +2524,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" @@ -1902,6 +2589,41 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-registry" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1953,13 +2675,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -1972,6 +2710,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -1984,6 +2728,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -1996,12 +2746,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -2014,6 +2776,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -2026,6 +2794,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -2038,6 +2812,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -2050,6 +2830,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.7.10" @@ -2059,6 +2845,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + [[package]] name = "writeable" version = "0.6.1" diff --git a/Cargo.toml b/Cargo.toml index 080d748..8d53b77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ indicatif = { version = "0.17" } futures-util = "0.3" anyhow = "1.0.98" clap = { version = "4.5.37", features = ["derive"] } +reqwest = { version = "0.12.19", features = ["stream", "gzip"] } +flate2 = "1.1.1" [profile.release] strip = true diff --git a/README.md b/README.md index e90d972..50b0770 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ ServerSeekerV2 is written in Rust allowing it to be blazingly fast and memory sa - Player and mod tracking. Find servers that have specific players online or servers running specific forge mods. (or both at the same time!) - Self Hostable. Host your own scanning instance and find your own servers! (See below for warnings against running this on a residential network) +- Country tracking. If enabled, tracks which country and Autonomous System a server is from. # For people just looking to not be scanned anymore diff --git a/config-example.toml b/config-example.toml index 35f0b1c..830ded6 100644 --- a/config-example.toml +++ b/config-example.toml @@ -8,12 +8,6 @@ port_range_start = 25565 # End of port range port_range_end = 25565 -[player_tracking] -# Enable player tracking? -enabled = false -# List of players to track -players = [""] - [masscan] # Location of masscans config file config_file = "masscan.conf" @@ -28,4 +22,18 @@ table = "postgres" # Database user user = "postgres" # Database user password -password = "SuperSuperSecretPassword" \ No newline at end of file +password = "SuperSuperSecretPassword" + +[player_tracking] +# Enable player tracking? +enabled = false +# List of players to track +players = [""] + +[country_tracking] +# Enable tracking of country information (requires a valid https://ipinfo.io account token) +enabled = false +# How often to update the country information table in hours (minimum 24 hours) +update_frequency = 48 +# IPInfo token +ipinfo_token = ":3" \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 0633b66..ed3199f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,15 +4,16 @@ use std::fs::File; use std::io::{ErrorKind, Read}; use tracing::error; -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Clone, Debug)] pub struct Config { pub database: Database, - pub player_tracking: PlayerTracking, pub scanner: ScannerConfig, pub masscan: Masscan, + pub player_tracking: PlayerTracking, + pub country_tracking: CountryTracking, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Clone, Debug)] pub struct Database { pub host: String, pub port: u16, @@ -21,13 +22,7 @@ pub struct Database { pub password: String, } -#[derive(Deserialize, Debug)] -pub struct PlayerTracking { - pub enabled: bool, - pub players: Vec, -} - -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Clone, Debug)] pub struct ScannerConfig { pub repeat: bool, pub scan_delay: u64, @@ -35,11 +30,24 @@ pub struct ScannerConfig { pub port_range_end: u16, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Clone, Debug)] pub struct Masscan { pub config_file: String, } +#[derive(Deserialize, Clone, Debug)] +pub struct PlayerTracking { + pub enabled: bool, + pub players: Vec, +} + +#[derive(Deserialize, Clone, Debug)] +pub struct CountryTracking { + pub enabled: bool, + pub update_frequency: u64, + pub ipinfo_token: String, +} + impl Default for Config { fn default() -> Self { Config { @@ -50,10 +58,6 @@ impl Default for Config { user: "postgres".to_string(), password: "password".to_string(), }, - player_tracking: PlayerTracking { - enabled: false, - players: vec![], - }, scanner: ScannerConfig { repeat: true, scan_delay: 60, @@ -63,6 +67,15 @@ impl Default for Config { masscan: Masscan { config_file: "masscan.conf".to_string(), }, + player_tracking: PlayerTracking { + enabled: false, + players: vec![], + }, + country_tracking: CountryTracking { + enabled: false, + update_frequency: 48, + ipinfo_token: "".to_string(), + }, } } } diff --git a/src/country_tracking.rs b/src/country_tracking.rs new file mode 100644 index 0000000..5bd3b24 --- /dev/null +++ b/src/country_tracking.rs @@ -0,0 +1,199 @@ +use crate::config::Config; +use anyhow::bail; +use flate2::read::GzDecoder; +use futures_util::StreamExt; +use indicatif::{ProgressBar, ProgressIterator, ProgressStyle}; +use serde::Deserialize; +use sqlx::types::ipnet::IpNet; +use sqlx::PgPool; +use std::fs::File; +use std::io::{Read, Write}; +use std::str::FromStr; +use std::time::Duration; +use tracing::{debug, info}; + +const DOWNLOAD_URL: &str = "https://ipinfo.io/data/ipinfo_lite.json.gz?token="; + +#[derive(Deserialize, Debug)] +struct CountryRow { + network: String, + country: String, + country_code: String, + asn: Option, + #[serde(rename = "as_name")] + company: Option, +} + +pub async fn country_tracking(pool: PgPool, config: Config) -> anyhow::Result<()> { + loop { + download_database(&config).await?; + insert_json_to_table(&pool).await?; + + // Sleep + tokio::time::sleep(Duration::from_secs( + config.country_tracking.update_frequency * 60 * 60, + )) + .await; + } +} + +async fn download_database(config: &Config) -> anyhow::Result<()> { + let url = format!("{}{}", DOWNLOAD_URL, config.country_tracking.ipinfo_token); + let response = reqwest::get(url).await?; + + // If response is OK write to file and unzip + if response.status().is_success() { + // Content length header is required for the progress bar + // IPInfo should always supply this + let content_length = match response.content_length() { + Some(len) => len, + None => bail!("Content-Length header was not set!"), + }; + + let mut downloaded: u64 = 0; + let mut output_file = File::create("ipinfo.json.gz")?; + let mut reader = response.bytes_stream(); + + let style = ProgressStyle::with_template( + "[{elapsed_precise}] [{bar:40.white/blue}] {bytes}/{total_bytes} {msg}", + ) + .expect("failed to create progress bar style") + .progress_chars("=>-"); + + let bar = ProgressBar::new(content_length).with_style(style); + bar.set_message("Downloading the latest version of the IPInfo database..."); + + while let Some(Ok(chunk)) = reader.next().await { + output_file.write_all(&chunk)?; + + // Update download bar position + let new = std::cmp::min(downloaded + (chunk.len() as u64), content_length); + downloaded = new; + bar.set_position(new); + } + + // Done + bar.finish_with_message("Finished!"); + info!("Decompressing output file..."); + + // Decompress file + let mut decoder = GzDecoder::new(File::open("ipinfo.json.gz")?); + let mut file = File::create("ipinfo.json")?; + let mut string = String::new(); + + // Write to output file + decoder.read_to_string(&mut string)?; + let mut output_file = File::create("ipinfo.json")?; + output_file.write_all(string.as_bytes())?; + file.flush()?; + + // Delete compressed file + std::fs::remove_file("ipinfo.json.gz")?; + + Ok(()) + } else { + bail!( + "IPInfo download failed: {} {:?}", + response.status(), + response.status().canonical_reason() + ) + } +} + +async fn parse_json_to_vec(string: String) -> serde_json::Result> { + serde_json::from_str(&format!( + "[{}]", + string + // Split at the end of every object + .split("}\n{") + // Skip all IPv6 netblocks + .map_while(|x| { + match x.contains("::") { + true => None, + false => Some(x), + } + }) + .map(|s| s.trim_matches(&['\n', '{', '}'][..])) + .map(|s| format!("{{{}}}", s)) + // Collect everything + .collect::>() + // Join everything with commas + .join(",") + )) +} + +pub async fn create_tables(pool: &PgPool) -> Result<(), sqlx::Error> { + // Create table + sqlx::query( + "CREATE TABLE IF NOT EXISTS countries ( + network CIDR, + country VARCHAR(255), + country_code VARCHAR(2), + asn VARCHAR(16), + company VARCHAR(255), + PRIMARY KEY(network) + )", + ) + .execute(pool) + .await?; + + // Use a GIST inet_ops index for the network column + // This allows for really fast lookup times from my testing + // + // Countries table needs to exist before we can create an index on it + sqlx::query("CREATE INDEX IF NOT EXISTS countries_table_index ON countries USING GIST (network inet_ops);") + .execute(pool) + .await?; + + Ok(()) +} + +async fn insert_json_to_table(pool: &PgPool) -> anyhow::Result<()> { + let mut file = File::open("ipinfo.json")?; + let mut string = String::new(); + file.read_to_string(&mut string)?; + + let json = parse_json_to_vec(string).await?; + info!("JSON Parsed successfully."); + + let mut transaction = pool.begin().await?; + + let style = ProgressStyle::with_template( + "[{elapsed_precise}] [{bar:40.white/blue}] {human_pos}/{human_len} {msg}", + ) + .expect("failed to create progress bar style") + .progress_chars("=>-"); + + let bar = ProgressBar::new(json.len() as u64).with_style(style); + bar.set_message("Inserting rows to countries table..."); + + for netblock in json.into_iter().progress_with(bar) { + if let Ok(cidr) = IpNet::from_str(&netblock.network) { + let result = sqlx::query( + "INSERT INTO countries VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (network, country, country_code) DO UPDATE + network = EXCLUDED.network, + country = EXCLUDED.country, + country_code = EXCLUDED.country_code, + asn = EXCLUDED.asn, + company = EXCLUDED.company", + ) + .bind(cidr) + .bind(netblock.country) + .bind(netblock.country_code) + .bind(netblock.asn) + .bind(netblock.company) + .execute(&mut *transaction) + .await; + + if let Err(e) = result { + debug!("Error while updating row in countries table {e}"); + } + }; + } + + transaction.commit().await?; + + info!("All done!"); + Ok(()) +} diff --git a/src/database.rs b/src/database.rs index e8b8ae4..e0c72e9 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,12 +1,29 @@ use crate::response::Server; use crate::utils::RunError; -use sqlx::postgres::PgQueryResult; +use sqlx::postgres::{PgQueryResult, PgRow}; use sqlx::types::ipnet::{IpNet, Ipv4Net}; use sqlx::types::Uuid; -use sqlx::{PgPool, Row}; +use sqlx::{FromRow, PgPool, Row}; use std::net::SocketAddrV4; use std::time::{SystemTime, UNIX_EPOCH}; +#[derive(Debug)] +struct AddressInfo { + country: String, + country_code: String, + asn: String, +} + +impl FromRow<'_, PgRow> for AddressInfo { + fn from_row(row: &PgRow) -> Result { + Ok(Self { + country: row.try_get("country")?, + country_code: row.try_get("country_code")?, + asn: row.try_get("asn")?, + }) + } +} + #[derive(Debug, Clone)] pub struct Database(pub PgPool); @@ -26,13 +43,20 @@ impl Database { } /// Deletes a server from the database - pub async fn delete_server(&self, address: IpNet) -> Result { + async fn delete_server(&self, address: IpNet) -> Result { sqlx::query("DELETE FROM servers WHERE address = $1") .bind(address) .execute(&self.0) .await } + async fn get_address_info(&self, address: &IpNet) -> Result { + sqlx::query_as("SELECT country, country_code, asn FROM countries WHERE $1 <<= network") + .bind(address) + .fetch_one(&self.0) + .await + } + /// Updates a single server in the database, this includes all mods /// and players that come with it. Will also remove a server from the /// database if it has requested to be removed @@ -52,6 +76,8 @@ impl Database { return Err(RunError::ServerOptOut)?; } + let address_information = self.get_address_info(&address).await?; + sqlx::query( "INSERT INTO servers ( address, @@ -67,7 +93,9 @@ impl Database { first_seen, last_seen, online_players, - max_players) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + max_players, + country, + asn) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (address, port) DO UPDATE SET software = EXCLUDED.software, version = EXCLUDED.version, @@ -79,7 +107,9 @@ impl Database { enforces_secure_chat = EXCLUDED.enforces_secure_chat, last_seen = EXCLUDED.last_seen, online_players = EXCLUDED.online_players, - max_players = EXCLUDED.max_players", + max_players = EXCLUDED.max_players, + country = EXCLUDED.country, + asn = EXCLUDED.asn", ) .bind(address) .bind(socket.port() as i32) @@ -99,6 +129,8 @@ impl Database { .bind(timestamp) .bind(server.players.online) .bind(server.players.max) + .bind(address_information.country_code) + .bind(address_information.asn) .execute(&self.0) .await?; diff --git a/src/main.rs b/src/main.rs index ec85b59..7d686c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ #![feature(let_chains)] mod config; +mod country_tracking; mod database; mod protocol; mod response; @@ -15,7 +16,7 @@ use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; use sqlx::ConnectOptions; use std::time::Duration; use tracing::log::LevelFilter; -use tracing::{error, info}; +use tracing::{debug, error, info}; #[derive(Parser, Debug)] #[clap(about = "Scans the internet for minecraft servers and indexes them")] @@ -64,6 +65,28 @@ async fn main() { .await .ok(); + if pool.is_none() { + error!("Failed to connect to database"); + std::process::exit(1); + } + + // Spawn a task to update the country info database everyday + if let Some(pool) = &pool + && config.country_tracking.enabled + { + if country_tracking::create_tables(pool).await.is_err() { + error!("failed to create tables"); + std::process::exit(1); + } + + debug!("Spawning country tracking task"); + + tokio::task::spawn(country_tracking::country_tracking( + pool.clone(), + config.clone(), + )); + } + Scanner::new() .config(config) .mode(arguments.mode) diff --git a/src/scanner.rs b/src/scanner.rs index 1946de6..17acd62 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -100,7 +100,7 @@ impl Scanner { let (tx, mut rx) = tokio::sync::mpsc::channel::(10); let mut stream = sqlx::query( - "SELECT (address - '0.0.0.0'::inet) AS address FROM servers ORDER BY last_seen ASC", + "SELECT (address - '0.0.0.0'::inet) AS address FROM servers ORDER BY last_seen DESC LIMIT 1000", ) .fetch(&self.database.0); @@ -135,9 +135,7 @@ impl Scanner { .expect("failed to create progress bar style") .progress_chars("=>-"); - let bar = - ProgressBar::new((total_servers as u64) * self.config.scanner.total_ports() as u64) - .with_style(style); + let bar = ProgressBar::new(1000).with_style(style); // Consume values from the receiver while let Some(socket) = rx.recv().await {