Browse Source

Remove old legacy cruft

develop
Olivier 'reivilibre' 3 months ago
parent
commit
dad85e6057
  1. 488
      Cargo.lock
  2. 6
      Cargo.toml
  3. 9
      datman/Cargo.toml
  4. 3
      datman/src/main.rs
  5. 345
      yama/src/xdef.rs
  6. 55
      yama/src/xpile.rs
  7. 413
      yama/src/xpile/local_pile.rs

488
yama/Cargo.lock → Cargo.lock

@ -1,39 +1,5 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "aes-ctr"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee"
dependencies = [
"aes-soft",
"aesni",
"ctr",
"stream-cipher",
]
[[package]]
name = "aes-soft"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d"
dependencies = [
"block-cipher-trait",
"byteorder",
"opaque-debug",
]
[[package]]
name = "aesni"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100"
dependencies = [
"block-cipher-trait",
"opaque-debug",
"stream-cipher",
]
[[package]]
name = "ahash"
version = "0.4.7"
@ -42,35 +8,30 @@ checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
[[package]]
name = "aho-corasick"
version = "0.7.10"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada"
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
dependencies = [
"memchr",
]
[[package]]
name = "anyhow"
version = "1.0.36"
version = "1.0.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479"
checksum = "15af2628f6890fe2609a3b91bef4c83450512802e59489f9c1cb1fa5df064a61"
[[package]]
name = "atty"
version = "0.2.13"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
[[package]]
name = "autocfg"
version = "1.0.1"
@ -85,46 +46,27 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "blake"
version = "2.0.0"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "200c5e8f05f9bf644dfcfde5d273bca1514e4283fd542500e755bbf7cee70b83"
checksum = "f76cff23583935d01f1d259e546bf988450648a644da066de24717fdd015aa81"
dependencies = [
"gcc",
"libc",
]
[[package]]
name = "block-cipher-trait"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c924d49bd09e7c06003acda26cd9742e796e34282ec6c1189404dee0c1f4774"
dependencies = [
"generic-array",
]
[[package]]
name = "byteorder"
version = "1.3.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
[[package]]
name = "c2-chacha"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
dependencies = [
"ppv-lite86",
]
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "cc"
version = "1.0.48"
version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f52a465a666ca3d838ebbf08b241383421412fe7ebb463527bba275526d89f76"
checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
dependencies = [
"jobserver",
"num_cpus",
]
[[package]]
@ -186,27 +128,19 @@ dependencies = [
[[package]]
name = "console"
version = "0.11.3"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c0994e656bba7b922d8dd1245db90672ffb701e684e45be58f20719d69abc5a"
checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"regex",
"terminal_size",
"termios",
"unicode-width",
"winapi",
"winapi-util",
]
[[package]]
name = "const_fn"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826"
[[package]]
name = "crc32fast"
version = "1.2.1"
@ -218,21 +152,22 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.4.0"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acec9a3b0b3559f15aee4f90746c4e5e293b701c0f7d3925d24e01645267b68c"
checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"
dependencies = [
"crossbeam-utils 0.7.0",
"crossbeam-utils 0.7.2",
"maybe-uninit",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.0"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.1",
"crossbeam-utils 0.8.5",
]
[[package]]
@ -243,18 +178,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils 0.8.1",
"crossbeam-utils 0.8.5",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.1"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
dependencies = [
"cfg-if 1.0.0",
"const_fn",
"crossbeam-utils 0.8.1",
"crossbeam-utils 0.8.5",
"lazy_static",
"memoffset",
"scopeguard",
@ -262,35 +196,28 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.7.0"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg 0.1.7",
"autocfg",
"cfg-if 0.1.10",
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.1"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
dependencies = [
"autocfg 1.0.1",
"cfg-if 1.0.0",
"lazy_static",
]
[[package]]
name = "ctr"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "022cd691704491df67d25d006fe8eca083098253c4d43516c2206479c58c6736"
dependencies = [
"block-cipher-trait",
"stream-cipher",
]
name = "datman"
version = "0.1.0"
[[package]]
name = "dirs-next"
@ -304,9 +231,9 @@ dependencies = [
[[package]]
name = "dirs-sys-next"
version = "0.1.1"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99de365f605554ae33f115102a02057d4fc18b01f3284d6870be0938743cfe7d"
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
dependencies = [
"libc",
"redox_users",
@ -352,13 +279,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastcdc"
version = "1.0.2"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e10bacfbef04c5c9c89635a18ccb6de1488d96e65393cb9b545747da1fa3a0c"
dependencies = [
"aes-ctr",
"byteorder",
]
checksum = "5afa29be46b12c8c380b997def8d1ac77c2665da93eb0a768fab0bf4db79333f"
[[package]]
name = "fs2"
@ -386,23 +309,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
[[package]]
name = "generic-array"
version = "0.12.3"
name = "getrandom"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [
"typenum",
"cfg-if 1.0.0",
"libc",
"wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.1.13"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"libc",
"wasi 0.7.0",
"wasi 0.10.0+wasi-snapshot-preview1",
]
[[package]]
@ -431,18 +356,18 @@ dependencies = [
[[package]]
name = "heck"
version = "0.3.1"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "hermit-abi"
version = "0.1.3"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
dependencies = [
"libc",
]
@ -458,11 +383,11 @@ dependencies = [
[[package]]
name = "indexmap"
version = "1.6.1"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b"
checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
dependencies = [
"autocfg 1.0.1",
"autocfg",
"hashbrown",
]
@ -498,13 +423,11 @@ dependencies = [
[[package]]
name = "jobserver"
version = "0.1.17"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b1d42ef453b30b7387e113da1c83ab1605d90c5b4e0eb8e96d016ed3b8c160"
checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
dependencies = [
"getrandom",
"libc",
"log",
]
[[package]]
@ -515,9 +438,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.81"
version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6"
[[package]]
name = "libsodium-sys"
@ -540,63 +463,43 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "libssh2-sys"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fcd5a428a31cbbfe059812d74f4b6cd3b9b7426c2bdaec56993c5365da1c328"
dependencies = [
"cc",
"libc",
"libz-sys",
"openssl-sys",
"pkg-config",
"vcpkg",
]
[[package]]
name = "libz-sys"
version = "1.0.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "lock_api"
version = "0.4.2"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.11"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
]
[[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "memchr"
version = "2.3.4"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
[[package]]
name = "memoffset"
version = "0.6.1"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9"
dependencies = [
"autocfg 1.0.1",
"autocfg",
]
[[package]]
@ -630,7 +533,7 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg 1.0.1",
"autocfg",
"num-traits",
]
@ -640,14 +543,14 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg 1.0.1",
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.11.1"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
dependencies = [
"hermit-abi",
"libc",
@ -659,25 +562,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
[[package]]
name = "opaque-debug"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
[[package]]
name = "openssl-sys"
version = "0.9.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "465d16ae7fc0e313318f7de5cecf57b2fbe7511fd213978b457e1c96ff46736f"
dependencies = [
"autocfg 0.1.7",
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "os_str_bytes"
version = "2.4.0"
@ -697,9 +581,9 @@ dependencies = [
[[package]]
name = "parking_lot_core"
version = "0.8.1"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0"
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
dependencies = [
"cfg-if 1.0.0",
"instant",
@ -711,15 +595,15 @@ dependencies = [
[[package]]
name = "pkg-config"
version = "0.3.17"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
[[package]]
name = "ppv-lite86"
version = "0.2.6"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "proc-macro-error"
@ -747,9 +631,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.7"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0319972dcae462681daf4da1adeeaa066e3ebd29c69be96c6abb1259d2ee2bcc"
checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
dependencies = [
"unicode-xid",
]
@ -762,20 +646,20 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quote"
version = "1.0.2"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.7.2"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom",
"getrandom 0.1.16",
"libc",
"rand_chacha",
"rand_core",
@ -784,11 +668,11 @@ dependencies = [
[[package]]
name = "rand_chacha"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"c2-chacha",
"ppv-lite86",
"rand_core",
]
@ -798,7 +682,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom",
"getrandom 0.1.16",
]
[[package]]
@ -812,11 +696,11 @@ dependencies = [
[[package]]
name = "rayon"
version = "1.5.0"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
dependencies = [
"autocfg 1.0.1",
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
@ -824,50 +708,52 @@ dependencies = [
[[package]]
name = "rayon-core"
version = "1.9.0"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
dependencies = [
"crossbeam-channel 0.5.0",
"crossbeam-channel 0.5.1",
"crossbeam-deque",
"crossbeam-utils 0.8.1",
"crossbeam-utils 0.8.5",
"lazy_static",
"num_cpus",
]
[[package]]
name = "redox_syscall"
version = "0.1.56"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.3.5"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"
checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
dependencies = [
"getrandom",
"getrandom 0.2.3",
"redox_syscall",
]
[[package]]
name = "regex"
version = "1.3.7"
version = "1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692"
checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
"thread_local",
]
[[package]]
name = "regex-syntax"
version = "0.6.17"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "rusqlite"
@ -913,9 +799,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "1.0.104"
version = "1.0.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449"
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
dependencies = [
"serde_derive",
]
@ -931,15 +817,21 @@ dependencies = [
[[package]]
name = "serde_derive"
version = "1.0.104"
version = "1.0.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64"
checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "shell-escape"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
[[package]]
name = "sled"
version = "0.34.6"
@ -948,7 +840,7 @@ checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
dependencies = [
"crc32fast",
"crossbeam-epoch",
"crossbeam-utils 0.8.1",
"crossbeam-utils 0.8.5",
"fs2",
"fxhash",
"libc",
@ -958,9 +850,9 @@ dependencies = [
[[package]]
name = "smallvec"
version = "1.5.1"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]]
name = "sodiumoxide"
@ -973,34 +865,20 @@ dependencies = [
"serde",
]
[[package]]
name = "ssh2"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "875fc74ffc41049e306a247f4191dd9c2ebfb7c5e0a6e4ddfdccacaba732e8aa"
dependencies = [
"bitflags",
"libc",
"libssh2-sys",
]
[[package]]
name = "sshish"
version = "0.1.0"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64e1ee53c6d5fad4125a5aec2006c51b2713af3fbba8b5ae73ea094c59498869"
checksum = "47cb7a759a4596d259849125fd9d58aceaea76fa521fed2540b06967b82827f9"
dependencies = [
"ssh2",
"shell-escape",
]
[[package]]
name = "stream-cipher"
version = "0.3.2"
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8131256a5896cabcf5eb04f4d6dacbe1aefda854b0d9896e09cb58829ec5638c"
dependencies = [
"generic-array",
]
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strsim"
@ -1010,9 +888,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "syn"
version = "1.0.12"
version = "1.0.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddc157159e2a7df58cd67b1cace10b8ed256a404fb0070593f137d8ba6bef4de"
checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
dependencies = [
"proc-macro2",
"quote",
@ -1027,32 +905,23 @@ checksum = "af547b166dd1ea4b472165569fc456cfb6818116f854690b0ff205e636523dab"
[[package]]
name = "termcolor"
version = "1.1.0"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
dependencies = [
"winapi-util",
]
[[package]]
name = "terminal_size"
version = "0.1.12"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8038f95fc7a6f351163f4b964af631bd26c9e828f7db085f2a84aca56f70d13b"
checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "termios"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2"
dependencies = [
"libc",
]
[[package]]
name = "textwrap"
version = "0.12.1"
@ -1064,33 +933,24 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.21"
version = "1.0.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42"
checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.21"
version = "1.0.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab"
checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thread_local"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
dependencies = [
"lazy_static",
]
[[package]]
name = "time"
version = "0.1.44"
@ -1104,28 +964,24 @@ dependencies = [
[[package]]
name = "toml"
version = "0.5.5"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf"
checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
dependencies = [
"serde",
]
[[package]]
name = "twox-hash"
version = "1.5.0"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56"
checksum = "04f8ab788026715fa63b31960869617cba39117e520eb415b0139543e325ab59"
dependencies = [
"cfg-if 0.1.10",
"rand",
"static_assertions",
]
[[package]]
name = "typenum"
version = "1.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9"
[[package]]
name = "unicode-segmentation"
version = "1.7.1"
@ -1134,15 +990,15 @@ checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]
name = "unicode-width"
version = "0.1.7"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479"
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]]
name = "unicode-xid"
version = "0.2.0"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "users"
@ -1161,21 +1017,21 @@ checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
[[package]]
name = "vcpkg"
version = "0.2.8"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa"
[[package]]
name = "vec_map"
version = "0.8.1"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version_check"
version = "0.9.2"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
[[package]]
name = "void"
@ -1185,9 +1041,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]]
name = "wasi"
version = "0.7.0"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasi"
@ -1197,9 +1053,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "winapi"
version = "0.3.8"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
@ -1235,8 +1091,8 @@ dependencies = [
"byteorder",
"chrono",
"clap",
"crossbeam-channel 0.4.0",
"crossbeam-utils 0.8.1",
"crossbeam-channel 0.4.4",
"crossbeam-utils 0.8.5",
"env_logger",
"fastcdc",
"glob",
@ -1263,18 +1119,18 @@ dependencies = [
[[package]]
name = "zstd"
version = "0.6.0+zstd.1.4.8"
version = "0.6.1+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4e44664feba7f2f1a9f300c1f6157f2d1bfc3c15c6f3cf4beabf3f5abe9c237"
checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "3.0.0+zstd.1.4.8"
version = "3.0.1+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9447afcd795693ad59918c7bbffe42fdd6e467d708f3537e3dc14dc598c573f"
checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c"
dependencies = [
"libc",
"zstd-sys",
@ -1282,12 +1138,10 @@ dependencies = [
[[package]]
name = "zstd-sys"
version = "1.4.19+zstd.1.4.8"
version = "1.4.20+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec24a9273d24437afb8e71b16f3d9a5d569193cccdb7896213b59f552f387674"
checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e"
dependencies = [
"cc",
"glob",
"itertools",
"libc",
]

6
Cargo.toml

@ -0,0 +1,6 @@
[workspace]
members = [
"yama",
"datman"
]

9
datman/Cargo.toml

@ -0,0 +1,9 @@
[package]
name = "datman"
version = "0.1.0"
authors = ["Olivier 'reivilibre' <olivier@librepush.net>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]

3
datman/src/main.rs

@ -0,0 +1,3 @@
fn main() {
println!("Hello, world!");
}

345
yama/src/xdef.rs

@ -1,345 +0,0 @@
use glob::{MatchOptions, Pattern};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashSet};
use std::convert::TryFrom;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;
pub type ChunkId = [u8; 32];
pub type XXHash = u64;
pub const XXH64_SEED: u64 = 424242;
pub type YamaResult<T> = Result<T, Box<dyn std::error::Error>>;
// yet unused: pub type YamaResultSend<T> = Result<T, Box<dyn std::error::Error + Send>>;
//pub type YamaResultA<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PointerData {
pub chunk_ref: RecursiveChunkRef,
pub parent_pointer: Option<String>,
pub uid_lookup: BTreeMap<u16, String>,
pub gid_lookup: BTreeMap<u16, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RecursiveChunkRef {
pub chunk_id: ChunkId,
pub depth: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct TreeNode {
#[serde(rename = "n")]
pub name: String,
//#[serde(flatten)]
#[serde(rename = "c")]
pub content: TreeNodeContent,
}
impl TreeNode {
/// whether the metadata invalidates these two nodes being equal, thus requiring a backup
pub fn metadata_invalidates(&self, other: &TreeNode, check_name: bool) -> bool {
if check_name {
self.name != other.name || self.content.metadata_invalidates(&other.content)
} else {
self.content.metadata_invalidates(&other.content)
}
}
/// Guarantees consistent visit order.
pub fn visit_mut<F>(
&mut self,
visitor: &mut F,
path_prefix: &str,
skip_components: u32,
) -> YamaResult<()>
where
F: FnMut(&mut Self, &str) -> YamaResult<()>,
{
let mut my_path_buf = String::new();
my_path_buf.push_str(path_prefix);
if skip_components == 0 {
if !my_path_buf.is_empty() {
my_path_buf.push('/');
}
my_path_buf.push_str(&self.name);
}
visitor(self, &my_path_buf)?;
if let TreeNodeContent::Directory { children, .. } = &mut self.content {
for child in children.iter_mut() {
let new_skip = if skip_components > 0 {
skip_components - 1
} else {
0
};
child.visit_mut(visitor, &my_path_buf, new_skip)?;
}
}
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(untagged)]
pub enum TreeNodeContent {
NormalFile {
// modification time in ms
#[serde(rename = "m")]
mtime: u64,
#[serde(flatten)]
#[serde(rename = "o")]
ownership: FilesystemOwnership,
#[serde(flatten)]
#[serde(rename = "p")]
permissions: FilesystemPermissions,
// TODO size: u64 or not
// can perhaps cache chunk-wise (but not sure.)
#[serde(rename = "c")]
content: RecursiveChunkRef,
},
Directory {
#[serde(flatten)]
#[serde(rename = "o")]
ownership: FilesystemOwnership,
#[serde(flatten)]
#[serde(rename = "p")]
permissions: FilesystemPermissions,
#[serde(rename = "C")]
children: Vec<TreeNode>,
},
SymbolicLink {
#[serde(flatten)]
#[serde(rename = "o")]
ownership: FilesystemOwnership,
#[serde(rename = "t")]
target: String,
},
// TODO is there any other kind of file we need to store?
Deleted,
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FilesystemOwnership {
pub uid: u16,
pub gid: u16,
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FilesystemPermissions {
pub mode: u32,
}
impl TreeNodeContent {
pub fn metadata_invalidates(&self, other: &TreeNodeContent) -> bool {
match self {
TreeNodeContent::NormalFile {
mtime,
ownership,
permissions,
..
} => {
if let TreeNodeContent::NormalFile {
mtime: other_mtime,
ownership: other_ownership,
permissions: other_permissions,
..
} = other
{
mtime != other_mtime
|| ownership != other_ownership
|| permissions != other_permissions
} else {
true
}
}
TreeNodeContent::Directory {
ownership,
permissions,
children,
} => {
if let TreeNodeContent::Directory {
ownership: other_ownership,
permissions: other_permissions,
children: other_children,
} = other
{
if ownership != other_ownership || permissions != other_permissions {
return true;
}
children
.iter()
.zip(other_children.iter())
.any(|(left, right)| left.metadata_invalidates(right, true))
} else {
true
}
}
TreeNodeContent::SymbolicLink { ownership, target } => {
if let TreeNodeContent::SymbolicLink {
ownership: other_ownership,
target: other_target,
} = other
{
ownership != other_ownership || target != other_target
} else {
true
}
}
TreeNodeContent::Deleted => {
// unreachable
false
}
}
}
}
pub struct Exclusions {
pub rules: Vec<ExclusionRule>,
}
impl Exclusions {
pub fn load(path: &Path) -> YamaResult<Exclusions> {
let file = File::open(path)?;
let bufreader = BufReader::new(file);
let mut rules = Vec::new();
for line in bufreader.lines() {
let line = line?;
let trim_line = line.trim();
if !trim_line.is_empty() {
rules.push(ExclusionRule::try_from(trim_line)?);
}
}
Ok(Exclusions { rules })
}
}
pub struct ExclusionRule {
pub glob: Pattern,
pub effect: Option<String>,
pub negated: bool,
}
impl TryFrom<&str> for ExclusionRule {
type Error = String;
fn try_from(value: &str) -> Result<Self, Self::Error> {
let mut effect = None;
let mut negated = false;
let mut glob_str = value;
let split: Vec<&str> = value.splitn(2, "?⇒").collect();
if split.len() == 2 {
// this is a conditional rule
glob_str = split[0].trim();
effect = Some(split[0].trim().to_owned());
}
if glob_str.starts_with("!") {
negated = true;
glob_str = &glob_str[1..];
}
Ok(ExclusionRule {
glob: Pattern::new(glob_str).map_err(|e| e.to_string())?,
effect,
negated,
})
}
}
impl Exclusions {
pub fn apply_to(&self, node: &mut TreeNode) -> YamaResult<()> {
self.apply_to_rec(node, "", &mut HashSet::new())
}
fn apply_to_rec(
&self,
node: &mut TreeNode,
path_rel: &str,
exclusions: &mut HashSet<String>,
) -> YamaResult<()> {
let match_options = MatchOptions {
case_sensitive: true,
require_literal_separator: true,
require_literal_leading_dot: false,
};
if let TreeNodeContent::Directory {
ref mut children, ..
} = node.content
{
let mut child_pathrel = String::new();
for rule in self.rules.iter() {
for child in children.iter() {
child_pathrel.clear();
child_pathrel.push_str(path_rel);
child_pathrel.push('/');
child_pathrel.push_str(&child.name);
if rule.glob.matches_with(&child_pathrel, match_options) {
if let Some(relative_effect) = &rule.effect {
let mut path_pieces: Vec<&str> =
child_pathrel.split("/").skip(1).collect();
let relative_pieces = relative_effect.split("/");
for (idx, relpiece) in relative_pieces.enumerate() {
match relpiece {
"" => {
if idx == 0 {
// this is an absolute path
// doubt we will use this feature much :/
path_pieces.clear();
path_pieces.push("");
}
}
"." => { /* nop */ }
".." => {
if path_pieces.len() > 1 {
path_pieces.pop();
}
}
other => {
path_pieces.push(other);
}
}
}
child_pathrel = path_pieces.join("/");
}
if rule.negated {
exclusions.remove(&child_pathrel);
} else {
exclusions.insert(child_pathrel.clone());
}
}
}
}
// filter out excluded children
children.retain(|child| {
child_pathrel.clear();
child_pathrel.push_str(path_rel);
child_pathrel.push('/');
child_pathrel.push_str(&child.name);
!exclusions.contains(&child_pathrel)
});
for child in children.iter_mut() {
child_pathrel.clear();
child_pathrel.push_str(path_rel);
child_pathrel.push('/');
child_pathrel.push_str(&child.name);
self.apply_to_rec(child, &child_pathrel, exclusions)?;
}
}
Ok(())
}
}

55
yama/src/xpile.rs

@ -1,55 +0,0 @@
use async_trait::async_trait;
use crate::def::{ChunkId, PointerData, XXHash, YamaResult};
mod interface {
use std::borrow::Cow;
use crate::def::{ChunkId, PointerData};
pub enum PileProcessorRequest<'a> {
GetDictionary,
GetChunk {
chunk_id: ChunkId,
},
PutChunk {
chunk_id: ChunkId,
chunk_data: Cow<'a, [u8]>,
},
/* DelChunk {
chunk_id:
}, */
XXHashChunk {
chunk_id: ChunkId,
},
GetPointer {
pointer_id: Cow<'a, str>,
},
PutPointer {
pointer_id: Cow<'a, str>,
pointer_data: PointerData,
},
// DelPointer,
ListPointers,
}
}
/// should be Cloneable to get another handle to the xpile.
#[async_trait]
pub trait Pile: Send + Sync + Clone {
/// Returns another handle to the Pile.
async fn get_dictionary(&self) -> YamaResult<Vec<u8>>;
async fn get_chunk(&self, chunk_id: &ChunkId) -> YamaResult<Option<Vec<u8>>>;
async fn put_chunk(&self, chunk_id: &ChunkId, data: &[u8], xxhash: &[u8]) -> YamaResult<()>;
async fn get_xxhash_chunk(&self, chunk_id: &ChunkId) -> YamaResult<Option<Vec<u8>>>;
async fn get_pointer(&self, pointer_id: &str) -> YamaResult<Option<PointerData>>;
async fn put_pointer(&self, pointer_id: &str, pointer_data: PointerData) -> YamaResult<()>;
async fn list_pointers(&self) -> YamaResult<Vec<String>>;
}
#[async_trait]
pub trait RawPile: Send + Sync + Clone {
async fn
}
pub mod local_pile;

413
yama/src/xpile/local_pile.rs

@ -1,413 +0,0 @@
use clap::crate_version;
use log::{info, trace, warn};
use std::fs::File;
use std::hash::Hasher;
use std::io::{Read, Write};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::sync::mpsc::{RecvTimeoutError, SyncSender};
use std::sync::{mpsc, Arc, Condvar, Mutex};
use std::time::Duration;
use std::{mem, thread};
use lmdb::{Cursor, Database, DatabaseFlags, Environment, EnvironmentFlags, Transaction};
use async_trait::async_trait;
use crate::def::{ChunkId, PointerData, XXHash, YamaResult, XXH64_SEED};
use crate::xpile::Pile;
use serde::{Deserialize, Serialize};
use std::thread::JoinHandle;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LocalPileManifest {
max_lmdb_size: usize,
yama_version: String,
}
impl Default for LocalPileManifest {
fn default() -> Self {
LocalPileManifest {
max_lmdb_size: 128 * 1024 * 1024 * 1024,
yama_version: String::from(crate_version!()),
}
}
}
#[derive(Clone)]
pub struct LocalPile {
directory: PathBuf,
dictionary: Arc<Vec<u8>>,
pub(crate) lmdb_env: Arc<Environment>,
pub(crate) lmdb_chunk_database: Database,
pub(crate) lmdb_pointer_database: Database,
lmdb_writing_thread_queue: Option<SyncSender<LmdbWriteOp>>,
lmdb_writing_thread_handle: Option<Arc<JoinHandle<()>>>,
}
impl Drop for LocalPile {
fn drop(&mut self) {
let mut opt_handle: Option<Arc<JoinHandle<()>>> = None;
self.lmdb_writing_thread_queue = None;
mem::swap(&mut self.lmdb_writing_thread_handle, &mut opt_handle);
if let Some(arc_join_handle) = opt_handle {
if let Ok(join_handle) = Arc::try_unwrap(arc_join_handle) {
//eprintln!("joining LMDB...");
if let Err(_) = join_handle.join() {
warn!("LMDB writer thread had an error when joining.");
}
//eprintln!("joined LMDB...");
}
}
}
}
impl LocalPile {
pub fn create(dir: &Path) -> YamaResult<()> {
let zstd_path = dir.join("zstd.dict");
eprintln!("Please add a Zstd dictionary to: {:?}", zstd_path);
let manifest_path = dir.join("yama.toml");
let manifest = LocalPileManifest::default();
{
let bytes = toml::to_vec(&manifest).expect("toml to_vec");
let mut file = File::create(manifest_path).expect("Failed to open yama manifest.");
file.write_all(&bytes)?;
// file automatically closed
}
let lmdb_env = Arc::new(
Environment::new()
.set_max_dbs(8)
.set_flags(EnvironmentFlags::NO_SUB_DIR | EnvironmentFlags::NO_TLS)
.set_map_size(manifest.max_lmdb_size)
.open(&dir.join("yama.lmdb"))
.expect("Failed to open LMDB env"),
);
let _lmdb_chunk_database = lmdb_env
.create_db(Some("yama_chunks"), DatabaseFlags::empty())
.expect("Failed to open yama_chunks");
let _lmdb_pointer_database = lmdb_env
.create_db(Some("yama_pointers"), DatabaseFlags::empty())
.expect("Failed to open yama_pointers");
//let lmdb_writing_thread_queue =
// make_lmdb_writer_thread(lmdb_env.clone(), lmdb_chunk_database, lmdb_pointer_database);
Ok(())
}
pub fn open(dir: &Path) -> YamaResult<Self> {
let zstd_path = dir.join("zstd.dict");
if !zstd_path.exists() {
return Err("No Zstd dictionary".into());
}
let manifest_path = dir.join("yama.toml");
if !manifest_path.exists() {
return Err("No Yama manifest".into());
}
let manifest = {
let mut file = File::open(manifest_path).expect("Failed to open yama manifest.");
let mut buf = Vec::new();
file.read_to_end(&mut buf).expect("Failed to read buf.");
toml::from_slice::<LocalPileManifest>(&buf)?
};
let dictionary = Arc::new({
let mut file = File::open(zstd_path)?;
let mut out = Vec::new();
file.read_to_end(&mut out)?;
out
});
let lmdb_env = Arc::new(
Environment::new()
.set_max_dbs(8)
.set_flags(EnvironmentFlags::NO_SUB_DIR | EnvironmentFlags::NO_TLS)
.set_map_size(manifest.max_lmdb_size)
.open(&dir.join("yama.lmdb"))
.expect("Failed to open LMDB env"),
);
let lmdb_chunk_database = lmdb_env
.open_db(Some("yama_chunks"))
.expect("Failed to open yama_chunks");
let lmdb_pointer_database = lmdb_env
.open_db(Some("yama_pointers"))
.expect("Failed to open yama_pointers");
let (writing_queue, join_handle) =
make_lmdb_writer_thread(lmdb_env.clone(), lmdb_chunk_database, lmdb_pointer_database);
Ok(LocalPile {
directory: dir.to_owned(),
dictionary,
lmdb_env,
lmdb_chunk_database,
lmdb_pointer_database,
lmdb_writing_thread_queue: Some(writing_queue),
lmdb_writing_thread_handle: Some(Arc::new(join_handle)),