Browse Source

Initial commit before I start messing around

rei/symlinks
Olivier 'reivilibre' 9 months ago
commit
328c99924c
  1. 23
      mypy.ini
  2. 1
      requirements.txt
  3. 0
      scone/__init__.py
  4. 55
      scone/__main__.py
  5. 0
      scone/common/__init__.py
  6. 172
      scone/common/chanpro.py
  7. 45
      scone/common/loader.py
  8. 42
      scone/common/misc.py
  9. 62
      scone/common/modeutils.py
  10. 20
      scone/common/pools.py
  11. 0
      scone/default/__init__.py
  12. 0
      scone/default/recipes/__init__.py
  13. 88
      scone/default/recipes/apt.py
  14. 322
      scone/default/recipes/filesystem.py
  15. 164
      scone/default/recipes/fridge.py
  16. 64
      scone/default/recipes/linux.py
  17. 94
      scone/default/recipes/postgres.py
  18. 82
      scone/default/recipes/python.py
  19. 114
      scone/default/recipes/systemd.py
  20. 0
      scone/default/steps/__init__.py
  21. 57
      scone/default/steps/basic_steps.py
  22. 7
      scone/default/steps/filesystem_steps.py
  23. 74
      scone/default/steps/fridge_steps.py
  24. 41
      scone/default/steps/linux_steps.py
  25. 0
      scone/default/utensils/__init__.py
  26. 149
      scone/default/utensils/basic_utensils.py
  27. 25
      scone/default/utensils/db_utensils.py
  28. 74
      scone/default/utensils/dynamic_dependencies.py
  29. 35
      scone/default/utensils/linux_utensils.py
  30. 201
      scone/head/__init__.py
  31. 132
      scone/head/cli/__init__.py
  32. 149
      scone/head/cli/freezer.py
  33. 26
      scone/head/cli/michelin.py
  34. 366
      scone/head/dependency_tracking.py
  35. 6
      scone/head/exceptions.py
  36. 192
      scone/head/kitchen.py
  37. 125
      scone/head/menu_reader.py
  38. 171
      scone/head/recipe.py
  39. 102
      scone/head/secrets.py
  40. 63
      scone/head/sshconn.py
  41. 27
      scone/head/utils.py
  42. 169
      scone/head/variables.py
  43. 27
      scone/sous/__init__.py
  44. 101
      scone/sous/__main__.py
  45. 26
      scone/sous/utensils.py
  46. 18
      scripts-dev/lint.sh
  47. 21
      setup.cfg
  48. 158
      setup.py
  49. 34
      tox.ini

23
mypy.ini

@ -0,0 +1,23 @@
[mypy]
check_untyped_defs = True
[mypy-cbor2]
ignore_missing_imports = True
[mypy-toposort]
ignore_missing_imports = True
[mypy-asyncssh]
ignore_missing_imports = True
[mypy-nacl.*]
ignore_missing_imports = True
[mypy-secretstorage]
ignore_missing_imports = True
[mypy-canonicaljson]
ignore_missing_imports = True
[mypy-asyncpg]
ignore_missing_imports = True

1
requirements.txt

@ -0,0 +1 @@
-e .

0
scone/__init__.py

55
scone/__main__.py

@ -0,0 +1,55 @@
# import asyncio
# import itertools
# import sys
# from typing import List
#
# from scone.head import Head, Recipe
# from scone.head.kitchen import Kitchen
# from scone.head.recipe import Preparation
# def main(args=None):
# if args is None:
# args = sys.argv[1:]
#
# if len(args) < 1:
# raise RuntimeError("Needs to be passed a sous config directory as 1st arg!")
#
# print("Am I a head?")
#
# head = Head.open(args[0])
#
# print(head.debug_info())
#
# recipes_by_sous = head.construct_recipes()
#
# all_recipes: List[Recipe] = list(
# itertools.chain.from_iterable(recipes_by_sous.values())
# )
#
# prepare = Preparation(all_recipes)
# order = prepare.prepare(head)
#
# for epoch, items in enumerate(order):
# print(f"----- Course {epoch} -----")
#
# for item in items:
# if isinstance(item, Recipe):
# print(f" > recipe {item}")
# elif isinstance(item, tuple):
# kind, ident, extra = item
# print(f" - we now have {kind} {ident} {dict(extra)}")
#
# print("Starting run")
#
# k = Kitchen(head)
#
# async def cook():
# for epoch, epoch_items in enumerate(order):
# print(f"Cooking Course {epoch} of {len(order)}")
# await k.run_epoch(epoch_items)
#
# asyncio.get_event_loop().run_until_complete(cook())
#
#
# if __name__ == "__main__":
# main()

0
scone/common/__init__.py

172
scone/common/chanpro.py

@ -0,0 +1,172 @@
import asyncio
import logging
import struct
import sys
from asyncio import Queue, Task
from asyncio.streams import FlowControlMixin, StreamReader, StreamWriter
from typing import Any, Dict, Optional
import attr
import cattr
import cbor2
SIZE_FORMAT = "!I"
logger = logging.getLogger(__name__)
class ChanPro:
def __init__(self, in_stream: StreamReader, out_stream: StreamWriter):
self._in = in_stream
self._out = out_stream
self._channels: Dict[int, "Channel"] = {}
self._listener: Optional[Task] = None
async def close(self) -> None:
# TODO cancel _listener?
pass
@staticmethod
async def open_from_stdio() -> "ChanPro":
loop = asyncio.get_event_loop()
reader = asyncio.StreamReader()
reader_protocol = asyncio.StreamReaderProtocol(reader)
await loop.connect_read_pipe(lambda: reader_protocol, sys.stdin.buffer)
writer_transport, writer_protocol = await loop.connect_write_pipe(
FlowControlMixin, sys.stdout.buffer
)
writer = StreamWriter(writer_transport, writer_protocol, None, loop)
return ChanPro(reader, writer)
async def _send_dict(self, dictionary: dict):
encoded = cbor2.dumps(dictionary)
encoded_len = struct.pack(SIZE_FORMAT, len(encoded))
self._out.write(encoded_len)
self._out.write(encoded)
await self._out.drain()
async def _recv_dict(self) -> dict:
size = struct.calcsize(SIZE_FORMAT)
encoded_len = await self._in.readexactly(size)
(length,) = struct.unpack(SIZE_FORMAT, encoded_len)
encoded = await self._in.readexactly(length)
return cbor2.loads(encoded)
def new_channel(self, number: int, desc: str):
if number in self._channels:
channel = self._channels[number]
raise ValueError(f"Channel {number} already in use ({channel}).")
channel = Channel(number, desc, self)
self._channels[number] = channel
return channel
async def send_message(self, channel: int, payload: Any):
await self._send_dict({"c": channel, "p": payload})
async def send_close(self, channel: int, reason: str = None):
# TODO extend with error indication capability (remote throw) ?
# TODO might want to wait until other end closed?
await self._send_dict({"c": channel, "close": True, "reason": reason})
def start_listening_to_channels(self, default_route: Optional["Channel"]):
async def channel_listener():
idx = 0
while True:
message = await self._recv_dict()
logger.debug("<message> %d %r", idx, message)
idx += 1
await self.handle_incoming_message(message, default_route=default_route)
self._listener = asyncio.create_task(
channel_listener() # py 3.8 , name="chanpro channel listener"
)
async def handle_incoming_message(
self, message: dict, default_route: Optional["Channel"] = None
):
if "c" not in message:
logger.warning("Received message without channel number.")
channel_num = message["c"]
channel = self._channels.get(channel_num)
if not channel:
if default_route:
await default_route._queue.put({"lost": message})
else:
logger.warning(
"Received message about non-existent channel number %r.",
channel_num,
)
return
# XXX todo send msg, what about shutdown too?
if "p" in message:
# payload on channel
await channel._queue.put(message["p"])
elif "close" in message:
channel._closed = True
await channel._queue.put(None)
else:
raise ValueError(f"Unknown channel message with keys {message.keys()}")
class Channel:
def __init__(self, number: int, desc: str, chanpro: ChanPro):
self.number = number
self.description = desc
self.chanpro = chanpro
self._queue: Queue[Any] = Queue()
self._closed = False
def __str__(self):
return f"Channel №{self.number} ({self.description})"
async def send(self, payload: Any):
if attr.has(payload.__class__):
payload = cattr.unstructure(payload)
await self.chanpro.send_message(self.number, payload)
async def recv(self) -> Any:
if self._queue.empty() and self._closed:
raise EOFError("Channel closed.")
item = await self._queue.get()
if item is None and self._queue.empty() and self._closed:
raise EOFError("Channel closed.")
return item
async def close(self, reason: str = None):
if not self._closed:
self._closed = True
await self._queue.put(None)
await self.chanpro.send_close(self.number, reason)
async def wait_close(self):
try:
await self.recv()
raise RuntimeError("Message arrived when expecting closure.")
except EOFError:
# expected
return
async def consume(self) -> Any:
"""
Consume the last item of the channel and assert closure.
The last item is returned.
"""
item = await self.recv()
await self.wait_close()
return item
class ChanProHead:
def __init__(self, chanpro: ChanPro, channel0: Channel):
self._chanpro = chanpro
self._channel0 = channel0
self._next_channel_id = 1
async def start_command_channel(self, command: str, payload: Any) -> Channel:
new_channel = self._chanpro.new_channel(self._next_channel_id, command)
self._next_channel_id += 1
await self._channel0.send(
{"nc": new_channel.number, "cmd": command, "pay": payload}
)
return new_channel

45
scone/common/loader.py

@ -0,0 +1,45 @@
import importlib
import pkgutil
from inspect import isclass
from typing import Any, Callable, Dict, Generic, Optional, TypeVar
T = TypeVar("T")
class ClassLoader(Generic[T]):
def __init__(self, clarse: Any, name_getter: Callable[[Any], Optional[str]]):
self._class = clarse
self._classes: Dict[str, Callable[[str, str, dict], T]] = dict()
self._name_getter = name_getter
def add_package_root(self, module_root: str):
module = importlib.import_module(module_root)
self._add_module(module)
# find subpackages
for mod in pkgutil.iter_modules(module.__path__): # type: ignore
if mod.ispkg:
self.add_package_root(module_root + "." + mod.name)
else:
submodule = importlib.import_module(module_root + "." + mod.name)
self._add_module(submodule)
def _add_module(self, module):
# find recipes
for name in dir(module):
item = getattr(module, name)
if isclass(item) and issubclass(item, self._class):
reg_name = self._name_getter(item)
if reg_name is not None:
self._classes[reg_name] = item
def get_class(self, name: str):
return self._classes.get(name)
def __str__(self) -> str:
lines = ["Generic Loader. Loaded stuff:"]
for recipe_name, recipe_class in self._classes.items():
lines.append(f" - {recipe_name} from {recipe_class.__module__}")
return "\n".join(lines)

42
scone/common/misc.py

@ -0,0 +1,42 @@
import os
import sys
from hashlib import sha256
def eprint(*args, **kwargs):
kwargs["file"] = sys.stderr
print(*args, **kwargs)
def sha256_dir(path: str) -> str:
items = {}
with os.scandir(path) as scandir:
for dir_entry in scandir:
if dir_entry.is_dir():
items[dir_entry.name] = sha256_dir(dir_entry.path)
else:
items[dir_entry.name] = sha256_file(dir_entry.path)
items_sorted = list(items.items())
items_sorted.sort()
buf = b""
for fname, fhash in items_sorted:
buf += fname.encode()
buf += b"\0"
buf += fhash.encode()
buf += b"\0"
return sha256_bytes(buf)
def sha256_file(path: str) -> str:
hasher = sha256(b"")
with open(path, "rb") as fread:
while True:
data = fread.read(8192 * 1024)
if not data:
break
hasher.update(data)
return hasher.hexdigest()
def sha256_bytes(data: bytes) -> str:
return sha256(data).hexdigest()

62
scone/common/modeutils.py

@ -0,0 +1,62 @@
import re
from typing import Union
# Opinionated default modes for personal use.
# Security conscious but also reasonable.
DEFAULT_MODE_FILE = 0o660
DEFAULT_MODE_DIR = 0o775
def parse_mode(mode_code: Union[str, int], directory: bool) -> int:
look_up = {"r": 0o4, "w": 0o2, "x": 0o1}
mults = {"u": 0o100, "g": 0o010, "o": 0o001, "a": 0o111}
mode = 0
if isinstance(mode_code, int):
return mode_code
pieces = mode_code.split(",")
for piece in pieces:
piecebits = 0
match = re.fullmatch(
r"(?P<affected>[ugoa]+)(?P<op>[-+=])(?P<value>[rwxXst]*)", piece
)
if match is None:
raise ValueError(f"Did not understand mode string {piece}")
affected = set(match.group("affected"))
op = match.group("op")
values = set(match.group("value"))
if "X" in values:
values.remove("X")
if directory:
values.add("x")
mult = 0
for affectee in affected:
mult |= mults[affectee]
for value in values:
if value in ("r", "w", "x"):
piecebits |= look_up[value] * mult
elif value == "s":
if "u" in affected:
piecebits |= 0o4000
if "g" in affected:
piecebits |= 0o2000
elif value == "t":
piecebits |= 0o1000
if op == "=":
# OR with piecebits allows setting suid, sgid and sticky.
mask = (mult * 0o7) | piecebits
mode &= ~mask
mode |= piecebits
elif op == "+":
mode |= piecebits
elif op == "-":
mode &= ~piecebits
else:
raise RuntimeError("op not [-+=].")
return mode

20
scone/common/pools.py

@ -0,0 +1,20 @@
from concurrent.futures.process import ProcessPoolExecutor
from concurrent.futures.thread import ThreadPoolExecutor
class Pools:
_instance = None
def __init__(self):
self.threaded = ThreadPoolExecutor()
self.process = ProcessPoolExecutor()
@staticmethod
def get():
if not Pools._instance:
Pools._instance = Pools()
return Pools._instance
def shutdown(self):
self.threaded.shutdown()
self.process.shutdown()

0
scone/default/__init__.py

0
scone/default/recipes/__init__.py

88
scone/default/recipes/apt.py

@ -0,0 +1,88 @@
from typing import Dict, List, Set, Tuple
from scone.default.utensils.basic_utensils import SimpleExec
from scone.head import Head, Recipe
from scone.head.kitchen import Kitchen
from scone.head.recipe import Preparation
from scone.head.utils import check_type
class AptInstallInternal(Recipe):
"""
Actually installs the packages; does it in a single batch for efficiency!
"""
_NAME = "apt-install.internal"
# TODO(extension, low): expand this into apt-install-now if we need
# the flexibility
def __init__(self, host: str, slug: str, args: dict, head: "Head"):
super().__init__(host, slug, args, head)
self.packages: Set[str] = set()
args["packages"] = self.packages
args[".source"] = ("@virtual", "apt-install-internal", "the one true AII")
def get_user(self, head: "Head") -> str:
return "root"
def prepare(self, preparation: Preparation, head: "Head") -> None:
super().prepare(preparation, head)
preparation.needs("apt-stage", "internal-install-packages")
preparation.needs("apt-stage", "repositories-declared")
preparation.provides("apt-stage", "packages-installed")
async def cook(self, kitchen: Kitchen) -> None:
# apt-installs built up the args to represent what was needed, so this
# will work as-is
kitchen.get_dependency_tracker()
if self.packages:
update = await kitchen.ut1areq(
SimpleExec(["apt-get", "-yq", "update"], "/"), SimpleExec.Result
)
if update.exit_code != 0:
raise RuntimeError(
f"apt update failed with err {update.exit_code}: {update.stderr!r}"
)
install_args = ["apt-get", "-yq", "install"]
install_args += list(self.packages)
install = await kitchen.ut1areq(
SimpleExec(install_args, "/"), SimpleExec.Result
)
if install.exit_code != 0:
raise RuntimeError(
f"apt install failed with err {install.exit_code}:"
f" {install.stderr!r}"
)
class AptPackage(Recipe):
_NAME = "apt-install"
internal_installers: Dict[Tuple[Head, str], AptInstallInternal] = {}
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
self.packages: List[str] = check_type(args["packages"], list)
def prepare(self, preparation: Preparation, head: Head) -> None:
super().prepare(preparation, head)
pair = (head, self.get_host())
if pair not in AptPackage.internal_installers:
install_internal = AptInstallInternal(self.get_host(), "internal", {}, head)
AptPackage.internal_installers[pair] = install_internal
preparation.subrecipe(install_internal)
preparation.provides("apt-stage", "internal-install-packages")
internal_installer = AptPackage.internal_installers.get(pair)
assert internal_installer is not None
internal_installer.packages.update(self.packages)
async def cook(self, kitchen: Kitchen) -> None:
# can't be tracked
kitchen.get_dependency_tracker().ignore()

322
scone/default/recipes/filesystem.py

@ -0,0 +1,322 @@
from pathlib import Path
from typing import List
from scone.common.modeutils import DEFAULT_MODE_DIR, parse_mode
from scone.default.steps.basic_steps import exec_no_fails
from scone.default.steps.filesystem_steps import depend_remote_file
from scone.default.utensils.basic_utensils import (
Chmod,
Chown,
MakeDirectory,
SimpleExec,
Stat,
)
from scone.default.utensils.dynamic_dependencies import HasChangedInSousStore
from scone.head import Head, Recipe
from scone.head.kitchen import Kitchen
from scone.head.recipe import Preparation
from scone.head.utils import check_type, check_type_opt
class DeclareFile(Recipe):
"""
Declares that a file already exists on the sous.
Maybe we will assert it in the future?
"""
_NAME = "declare-file"
def prepare(self, preparation: Preparation, head: Head):
preparation.provides("file", self._args["path"])
async def cook(self, kitchen: Kitchen):
# mark as tracked.
kitchen.get_dependency_tracker()
class DeclareDirectory(Recipe):
"""
Declares that a directory already exists on the sous.
Maybe we will assert it in the future?
"""
_NAME = "declare-dir"
def prepare(self, preparation: Preparation, head: Head):
preparation.provides("directory", self._args["path"])
async def cook(self, kitchen: Kitchen):
# mark as tracked.
kitchen.get_dependency_tracker()
class EnsureDirectory(Recipe):
"""
Makes a directory tree.
"""
_NAME = "directory"
def __init__(self, host: str, slug: str, args: dict, head: "Head"):
super().__init__(host, slug, args, head)
parents = args.get("parents", 0)
assert isinstance(parents, int)
path = args.get("path")
assert isinstance(path, str)
mode = args.get("mode", DEFAULT_MODE_DIR)
assert isinstance(mode, str) or isinstance(mode, int)
self.path = path
self.parents = parents
self.mode = parse_mode(mode, directory=True)
self._make: List[str] = []
self.targ_user = args.get("owner", self.get_user(head))
self.targ_group = args.get("group", self.targ_user)
def prepare(self, preparation: Preparation, head: "Head"):
super().prepare(preparation, head)
preparation.needs("os-user", self.targ_user)
preparation.needs("os-group", self.targ_group)
preparation.provides("directory", self.path)
self._make.append(self.path)
parent = Path(self.path).parent
for _ in range(self.parents):
self._make.append(str(parent))
preparation.provides("directory", str(parent))
parent = parent.parent
preparation.needs("directory", str(parent))
self._make.reverse()
async def cook(self, k: Kitchen):
for directory in self._make:
stat = await k.ut1a(Stat(directory), Stat.Result)
if stat is None:
# doesn't exist, make it
await k.ut0(MakeDirectory(directory, self.mode))
stat = await k.ut1a(Stat(directory), Stat.Result)
if stat is None:
raise RuntimeError("Directory vanished after creation!")
if stat.dir:
if (stat.user, stat.group) != (self.targ_user, self.targ_group):
# need to chown
await k.ut0(Chown(directory, self.targ_user, self.targ_group))
if stat.mode != self.mode:
await k.ut0(Chmod(directory, self.mode))
else:
raise RuntimeError("Already exists but not a dir: " + directory)
# mark as tracked.
k.get_dependency_tracker()
class ExtractTar(Recipe):
"""
Extracts a tar archive, expecting to get at least some files.
"""
_NAME = "tar-extract"
def __init__(self, host: str, slug: str, args: dict, head: "Head"):
super().__init__(host, slug, args, head)
self.tar = check_type(args.get("tar"), str)
self.dir = check_type(args.get("dir"), str)
self.expect_files = check_type(args.get("expects_files"), List[str])
def prepare(self, preparation: Preparation, head: "Head"):
super().prepare(preparation, head)
preparation.needs("file", self.tar)
preparation.needs("directory", self.dir)
for file in self.expect_files:
assert isinstance(file, str)
final = str(Path(self.dir, file))
preparation.provides("file", final)
async def cook(self, k: "Kitchen"):
res = await k.ut1areq(
SimpleExec(["tar", "xf", self.tar], self.dir), SimpleExec.Result
)
if res.exit_code != 0:
raise RuntimeError(
f"tar failed with ec {res.exit_code}; stderr = <<<"
f"\n{res.stderr.decode()}\n>>>"
)
for expect_relative in self.expect_files:
expect = str(Path(self.dir, expect_relative))
stat = await k.ut1a(Stat(expect), Stat.Result)
if stat is None:
raise RuntimeError(
f"tar succeeded but expectation failed; {expect!r} not found."
)
class RunScript(Recipe):
"""
Runs a script (such as an installation script).
"""
_NAME = "script-run"
def __init__(self, host: str, slug: str, args: dict, head: "Head"):
super().__init__(host, slug, args, head)
self.working_dir = check_type(args.get("working_dir"), str)
# relative to working dir
self.script = check_type(args.get("script"), str)
# todo other remote dependencies
# todo provided files as a result of the script exec
def prepare(self, preparation: Preparation, head: "Head"):
super().prepare(preparation, head)
final_script = str(Path(self.working_dir, self.script))
preparation.needs("file", final_script)
# TODO more needs
# TODO preparation.provides()
async def cook(self, kitchen: "Kitchen"):
final_script = str(Path(self.working_dir, self.script))
await depend_remote_file(final_script, kitchen)
class CommandOnChange(Recipe):
"""
Runs a command when at least one file listed has changed on the remote.
"""
_NAME = "command-on-change"
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
self.purpose = check_type(args.get("purpose"), str)
self.command = check_type(args.get("command"), list)
self.watching = check_type(args.get("files"), list)
self.working_dir = check_type(args.get("working_dir", "/"), str)
def prepare(self, preparation: Preparation, head: Head) -> None:
super().prepare(preparation, head)
for file in self.watching:
preparation.needs("file", file)
async def cook(self, kitchen: Kitchen) -> None:
kitchen.get_dependency_tracker().ignore()
changed = await kitchen.ut1(HasChangedInSousStore(self.purpose, self.watching))
if changed:
result = await kitchen.ut1areq(
SimpleExec(self.command, self.working_dir), SimpleExec.Result
)
if result.exit_code != 0:
raise RuntimeError(
f"exit code not 0 ({result.exit_code}), {result.stderr!r}"
)
class GitCheckout(Recipe):
_NAME = "git"
# TODO(correctness): branches can change (tags too), but this will still
# declare SAFE_TO_SKIP. Perhaps we want to stop that unless you opt out?
# But oh well for now.
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
self.repo_src = check_type(args.get("src"), str)
self.dest_dir = check_type(args.get("dest"), str)
self.ref = check_type_opt(args.get("ref"), str)
self.branch = check_type_opt(args.get("branch"), str)
if not (self.ref or self.branch):
raise ValueError("Need to specify 'ref' or 'branch'")
if self.ref and self.branch:
raise ValueError("Can't specify both 'ref' and 'branch'.")
# should end with / if it's a dir
self.expect: List[str] = check_type(args.get("expect", []), list)
self.submodules = check_type(args.get("submodules", False), bool)
def prepare(self, preparation: Preparation, head: Head) -> None:
super().prepare(preparation, head)
parent = str(Path(self.dest_dir).parent)
preparation.needs("directory", parent)
preparation.provides("directory", self.dest_dir)
for expected in self.expect:
expected_path_str = str(Path(self.dest_dir, expected))
if expected.endswith("/"):
preparation.provides("directory", expected_path_str)
else:
preparation.provides("file", expected_path_str)
async def cook(self, k: Kitchen) -> None:
# no non-arg dependencies
k.get_dependency_tracker()
stat = await k.ut1a(Stat(self.dest_dir), Stat.Result)
if stat is None:
# doesn't exist; git init it
await exec_no_fails(
k, ["git", "init", self.dest_dir], "/"
)
stat = await k.ut1a(Stat(self.dest_dir), Stat.Result)
if stat is None:
raise RuntimeError("Directory vanished after creation!")
if not stat.dir:
raise RuntimeError("Already exists but not a dir: " + self.dest_dir)
# add the remote, removing it first to ensure it's what we want
# don't care if removing fails
await k.ut1areq(SimpleExec(["git", "remote", "remove", "scone"], self.dest_dir), SimpleExec.Result)
await exec_no_fails(
k, ["git", "remote", "add", "scone", self.repo_src], self.dest_dir
)
# fetch the latest from the remote
await exec_no_fails(
k, ["git", "fetch", "scone"], self.dest_dir
)
# figure out what ref we want to use
# TODO(performance): fetch only this ref?
ref = self.ref or f"scone/{self.branch}"
# switch to that ref
await exec_no_fails(
k, ["git", "switch", "--detach", ref], self.dest_dir
)
# if we use submodules
if self.submodules:
await exec_no_fails(
k, ["git", "submodule", "update", "--init", "--recursive"], self.dest_dir
)
for expected in self.expect:
expected_path_str = str(Path(self.dest_dir, expected))
# TODO(performance, low): parallelise these
stat = await k.ut1a(Stat(expected_path_str), Stat.Result)
if not stat:
raise RuntimeError(f"expected {expected_path_str} to exist but it did not")
if stat.dir and not expected.endswith("/"):
raise RuntimeError(f"expected {expected_path_str} to exist as a file but it is a dir")
if not stat.dir and expected.endswith("/"):
raise RuntimeError(f"expected {expected_path_str} to exist as a dir but it is a file")

164
scone/default/recipes/fridge.py

@ -0,0 +1,164 @@
import asyncio
from asyncio import Future
from pathlib import Path
from typing import Dict, cast
from urllib.parse import urlparse
from urllib.request import urlretrieve
from scone.common.misc import sha256_file
from scone.common.modeutils import DEFAULT_MODE_FILE, parse_mode
from scone.default.steps import fridge_steps
from scone.default.steps.fridge_steps import FridgeMetadata, load_and_transform, SUPERMARKET_RELATIVE
from scone.default.utensils.basic_utensils import WriteFile, Chown
from scone.head import Head
from scone.head.kitchen import Kitchen
from scone.head.recipe import Preparation, Recipe
from scone.head.utils import check_type
class FridgeCopy(Recipe):
"""
Declares that a file should be copied from the head to the sous.
"""
_NAME = "fridge-copy"
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
fp = fridge_steps.search_in_fridge(head, args["src"])
if fp is None:
raise ValueError(f"Cannot find {args['src']} in the fridge.")
unextended_path_str, meta = fridge_steps.decode_fridge_extension(str(fp))
unextended_path = Path(unextended_path_str)
dest = args["dest"]
if not isinstance(dest, str):
raise ValueError("No destination provided or wrong type.")
if dest.endswith("/"):
self.destination: Path = Path(args["dest"], unextended_path.parts[-1])
else:
self.destination = Path(args["dest"])
mode = args.get("mode", DEFAULT_MODE_FILE)
assert isinstance(mode, str) or isinstance(mode, int)
self.fridge_path: str = args["src"]
self.real_path: Path = fp
self.fridge_meta: FridgeMetadata = meta
self.mode = parse_mode(mode, directory=False)
def prepare(self, preparation: Preparation, head: Head) -> None:
super().prepare(preparation, head)
preparation.provides("file", str(self.destination))
preparation.needs("directory", str(self.destination.parent))
async def cook(self, k: Kitchen) -> None:
data = await load_and_transform(
k, self.fridge_meta, self.real_path, self.get_host()
)
dest_str = str(self.destination)
chan = await k.start(WriteFile(dest_str, self.mode))
await chan.send(data)
await chan.send(None)
if await chan.recv() != "OK":
raise RuntimeError(f"WriteFail failed on fridge-copy to {self.destination}")
# this is the wrong thing
# hash_of_data = sha256_bytes(data)
# k.get_dependency_tracker().register_remote_file(dest_str, hash_of_data)
await k.get_dependency_tracker().register_fridge_file(
self.fridge_path, self.real_path
)
class Supermarket(Recipe):
"""
Downloads an asset (cached if necessary) and copies to sous.
"""
_NAME = "supermarket"
# dict of target path → future that will complete when it's downloaded
in_progress: Dict[str, Future] = dict()
def __init__(self, host: str, slug: str, args: dict, head: "Head"):
super().__init__(host, slug, args, head)
self.url = args.get("url")
assert isinstance(self.url, str)
self.sha256 = check_type(args.get("sha256"), str).lower()
dest = args["dest"]
if not isinstance(dest, str):
raise ValueError("No destination provided or wrong type.")
if dest.endswith("/"):
file_basename = urlparse(self.url).path.split("/")[-1]
self.destination: Path = Path(args["dest"], file_basename).resolve()
else:
self.destination = Path(args["dest"]).resolve()
self.owner = check_type(args.get("owner", self.get_user(head)), str)
self.group = check_type(args.get("group", self.owner), str)
mode = args.get("mode", DEFAULT_MODE_FILE)
assert isinstance(mode, str) or isinstance(mode, int)
self.mode = parse_mode(mode, directory=False)
def prepare(self, preparation: Preparation, head: "Head"):
super().prepare(preparation, head)
preparation.provides("file", str(self.destination))
async def cook(self, kitchen: "Kitchen"):
# need to ensure we download only once, even in a race…
supermarket_path = Path(kitchen.head.directory, SUPERMARKET_RELATIVE, self.sha256)
if self.sha256 in Supermarket.in_progress:
await Supermarket.in_progress[self.sha256]
elif not supermarket_path.exists():
note = f"""
Scone Supermarket
This file corresponds to {self.url}
Downloaded by {self}
""".strip()
Supermarket.in_progress[self.sha256] = cast(
Future,
asyncio.get_running_loop().run_in_executor(
kitchen.head.pools.threaded,
self._download_file,
self.url,
str(supermarket_path),
self.sha256,
note
),
)
# TODO(perf): load file in another thread
with open(supermarket_path, "r") as fin:
data = fin.read()
chan = await kitchen.start(WriteFile(str(self.destination), self.mode))
await chan.send(data)
await chan.send(None)
if await chan.recv() != "OK":
raise RuntimeError(f"WriteFail failed on supermarket to {self.destination}")
await kitchen.ut0(Chown(str(self.destination), self.owner, self.group))
@staticmethod
def _download_file(url: str, dest_path: str, check_sha256: str, note: str):
urlretrieve(url, dest_path)
real_sha256 = sha256_file(dest_path)
if real_sha256 != check_sha256:
raise RuntimeError(
f"sha256 hash mismatch {real_sha256} != {check_sha256} (wanted)"
)
with open(dest_path + ".txt", "w") as fout:
# leave a note so we can find out what this is if we need to.
fout.write(note)

64
scone/default/recipes/linux.py

@ -0,0 +1,64 @@
import crypt
import logging
from typing import Optional
from scone.default.steps import linux_steps
from scone.default.utensils.linux_utensils import GetPasswdEntry
from scone.head import Head, Recipe
from scone.head.kitchen import Kitchen
from scone.head.recipe import Preparation
from scone.head.utils import check_type, check_type_opt
logger = logging.getLogger(__name__)
class LinuxUser(Recipe):
_NAME = "os-user"
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
if slug[0] == "@":
raise ValueError("os-user should be used like [os-user.username].")
self.user_name = slug
self.make_group = check_type(args.get("make_group", True), bool)
self.make_home = check_type(args.get("make_home", True), bool)
self.home: Optional[str] = check_type_opt(args.get("home"), str)
self.password: Optional[str] = check_type_opt(args.get("password"), str)
def prepare(self, preparation: Preparation, head: "Head") -> None:
super().prepare(preparation, head)
preparation.provides("os-user", self.user_name)
if self.make_group:
preparation.provides("os-group", self.user_name)
async def cook(self, kitchen: Kitchen) -> None:
# TODO(documentation): note this does not update users
# acknowledge tracking
kitchen.get_dependency_tracker()
if self.password:
password_hash: Optional[str] = crypt.crypt(self.password)
else:
password_hash = None
pwd_entry = await kitchen.ut1a(
GetPasswdEntry(self.user_name), GetPasswdEntry.Result
)
if pwd_entry:
logger.warning(
"Not updating existing os-user '%s' as it exists already and "
"modifications could be dangerous in any case. Modification "
"support may be implemented in the future.",
self.user_name,
)
else:
# create the user fresh
await linux_steps.create_linux_user(
kitchen,
self.user_name,
password_hash,
self.make_home,
self.make_group,
self.home,
)

94
scone/default/recipes/postgres.py

@ -0,0 +1,94 @@
from scone.default.utensils.db_utensils import PostgresTransaction
from scone.head import Head, Recipe
from scone.head.kitchen import Kitchen
from scone.head.recipe import Preparation
from scone.head.utils import check_type
class PostgresDatabase(Recipe):
_NAME = "pg-db"
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
self.database_name = slug
self.owner = check_type(args.get("owner"), str)
self.encoding = args.get("encoding", "utf8")
self.collate = args.get("collate", "en_GB.utf8")
self.ctype = args.get("ctype", "en_GB.utf8")
self.template = args.get("template", "template0")
def prepare(self, preparation: Preparation, head: Head) -> None:
super().prepare(preparation, head)
# todo
async def cook(self, kitchen: Kitchen) -> None:
ch = await kitchen.start(PostgresTransaction("postgres"))
await ch.send(
(
"SELECT 1 AS count FROM pg_catalog.pg_database WHERE datname = ?;",
self.database_name,
)
)
dbs = await ch.recv()
if len(dbs) > 0 and dbs[0]["count"] == 1:
await ch.send(None)
await ch.wait_close()
return
q = f"""
CREATE DATABASE {self.database_name}
WITH OWNER {self.owner}
ENCODING {self.encoding}
LC_COLLATE {self.collate}
LC_CTYPE {self.ctype}
TEMPLATE {self.template};
"""
await ch.send((q,))
res = await ch.recv()
if len(res) != 0:
raise RuntimeError("expected empty result set.")
await ch.send(None)
await ch.wait_close()
class PostgresUser(Recipe):
_NAME = "pg-user"
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
self.user_name = slug
self.password = check_type(args.get("password"), str)
def prepare(self, preparation: Preparation, head: Head) -> None:
super().prepare(preparation, head)
# todo
async def cook(self, kitchen: Kitchen) -> None:
ch = await kitchen.start(PostgresTransaction("postgres"))
await ch.send(
(
"SELECT 1 AS count FROM pg_catalog.pg_user WHERE usename = ?;",
self.user_name,
)
)
dbs = await ch.recv()
if len(dbs) > 0 and dbs[0]["count"] == 1:
await ch.send(None)
await ch.wait_close()
return
q = f"""
CREATE ROLE {self.user_name}
WITH PASSWORD ?
LOGIN;
"""
await ch.send((q, self.password))
res = await ch.recv()
if len(res) != 0:
raise RuntimeError("expected empty result set.")
await ch.send(None)
await ch.wait_close()

82
scone/default/recipes/python.py

@ -0,0 +1,82 @@
from pathlib import Path
from typing import Tuple, List
from scone.default.recipes.apt import AptPackage
from scone.default.steps.basic_steps import exec_no_fails
from scone.default.steps.filesystem_steps import depend_remote_file
from scone.head import Head, Recipe
from scone.head.kitchen import Kitchen
from scone.head.recipe import Preparation
from scone.head.utils import check_type
class PythonVenv(Recipe):
"""
Creates a Python virtualenv with a specified set of requirements.
Note: using a directory as a dependency can be inefficient as dir SHA256
will be performed to check it has not changed.
"""
_NAME = "python-venv"
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
self.dir = check_type(args.get("dir"), str)
self.interpreter = check_type(args.get("interpreter"), str)
# list of flags. Current supported:
# git (local git repo — track hash by git commit hash), dir, -r
self.install: List[Tuple[str, List[str]]] = []
install_plaintext = check_type(args.get("install"), list)
for install_line in install_plaintext:
parts = install_line.split(" ")
self.install.append((parts[-1], parts[0:-1]))
self.no_apt_install = check_type(args.get("_no_apt_install", False), bool)
# TODO(sdists)
def prepare(self, preparation: Preparation, head: Head):
super().prepare(preparation, head)
preparation.needs("dir", str(Path(self.dir).parent))
for name, flags in self.install:
if "-r" in flags:
preparation.needs("file", name)
elif "git" in flags or "dir" in flags:
preparation.needs("dir", name)
final_script = str(Path(self.dir, "bin/python"))
preparation.provides("file", str(final_script))
if not self.no_apt_install:
preparation.subrecipe(
AptPackage(
self.get_host(), "@venv-apt", {"packages": ["python3-venv"]}, head
)
)
preparation.needs("apt-stage", "packages-installed")
async def cook(self, kitchen: Kitchen):
dt = kitchen.get_dependency_tracker()
await exec_no_fails(
kitchen, [self.interpreter, "-m", "venv", self.dir], "/"
)
install_args = []
for name, flags in self.install:
if "-r" in flags:
install_args.append("-r")
await depend_remote_file(name, kitchen)
elif "dir" in flags or "git" in flags:
# TODO(perf, dedup): custom dynamic dependency types; git
# dependencies and sha256_dir dependencies.
dt.ignore()
install_args.append(name)
await exec_no_fails(
kitchen, [self.dir + "/bin/pip", "install"] + install_args, "/"
)

114
scone/default/recipes/systemd.py

@ -0,0 +1,114 @@
from typing import Dict
from scone.default.recipes.filesystem import CommandOnChange
from scone.default.utensils.basic_utensils import SimpleExec
from scone.head import Head, Recipe
from scone.head.kitchen import Kitchen
from scone.head.recipe import Preparation
from scone.head.utils import check_type, check_type_opt
class SystemdUnit(Recipe):
"""
Shorthand for a system unit. Metarecipe.
"""
_NAME = "systemd"
daemon_reloaders: Dict[str, CommandOnChange] = {}
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
self.unit_name = slug if "." in slug else slug + ".service"
self.at = check_type(args.get("at"), str)
self.enabled = check_type_opt(args.get("enabled"), bool)
self.restart_on = check_type_opt(args.get("restart_on"), list)
def prepare(self, preparation: Preparation, head: Head) -> None:
super().prepare(preparation, head)
preparation.provides("systemd-unit", self.unit_name)
preparation.needs("systemd-stage", "daemon-reloaded")
if self.enabled is not None:
enable_recipe = SystemdEnabled(
self.get_host(),
self.unit_name,
{"enabled": self.enabled, "at": self.at, ".user": "root"},
head,
)
preparation.subrecipe(enable_recipe)
preparation.needs("systemd-stage", "enabled")
daemon_reloader = SystemdUnit.daemon_reloaders.get(self.get_host(), None)
if not daemon_reloader:
# TODO this should be replaced with a dedicated command which provides
# those units.
daemon_reloader = CommandOnChange(
self.get_host(),
"systemd-internal",
{
"purpose": "systemd.daemon_reload",
"command": ["systemctl", "daemon-reload"],
"files": [],
".user": "root",
},
head,
)
preparation.subrecipe(daemon_reloader)
file_list = getattr(daemon_reloader, "_args")["files"]
file_list.append(self.at)
if self.restart_on:
service_reloader = CommandOnChange(
self.get_host(),
"systemd-internal",
{
"purpose": "systemd.unit_reload",
"command": ["systemctl", "reload", self.unit_name],
"files": self.restart_on + [self.at],
".user": "root",
},
head,
)
preparation.subrecipe(service_reloader)
async def cook(self, kitchen: Kitchen) -> None:
# metarecipes don't do anything.
kitchen.get_dependency_tracker().ignore()
class SystemdEnabled(Recipe):
"""
Sets the enabled state of the systemd unit.
"""
_NAME = "systemd-enabled"
def __init__(self, host: str, slug: str, args: dict, head: Head):
super().__init__(host, slug, args, head)
self.unit_name = slug if "." in slug else slug + ".service"
self.at = check_type(args.get("at"), str)
self.enabled = check_type_opt(args.get("enabled"), bool)
def prepare(self, preparation: Preparation, head: Head) -> None:
super().prepare(preparation, head)
preparation.needs("file", self.at)
preparation.needs("systemd-stage", "daemon-reloaded")
async def cook(self, kitchen: Kitchen) -> None:
kitchen.get_dependency_tracker()
result = await kitchen.ut1areq(
SimpleExec(
["systemctl", "enable" if self.enabled else "disable", self.unit_name],
"/",
),
SimpleExec.Result,
)
if result.exit_code != 0:
raise RuntimeError(
f"Failed to en/disable {self.unit_name}: {result.stderr.decode()}"
)

0
scone/default/steps/__init__.py

57
scone/default/steps/basic_steps.py

@ -0,0 +1,57 @@
from pathlib import PurePath
from typing import List, Optional, Union
from scone.default.utensils.basic_utensils import SimpleExec
from scone.head import Recipe
from scone.head.exceptions import CookingError
from scone.head.kitchen import Kitchen, current_recipe
class ExecutionFailure(CookingError):
"""
A command failed.
"""
def __init__(
self,
args: List[str],
working_dir: str,
sous: str,
user: str,
result: SimpleExec.Result,
):
stderr = result.stderr.decode().replace("\n", "\n ")
message = (
f"Command failed on {sous} (user {user}) in {working_dir}.\n"
f"The command was: {args}\n"
f"Stderr was:\n {stderr}"
)
super().__init__(message)
async def exec_no_fails(
kitchen: Kitchen, args: List[str], working_dir: Union[str, PurePath]
) -> SimpleExec.Result:
if not isinstance(working_dir, str):
working_dir = str(working_dir)
result = await kitchen.start_and_consume_attrs(
SimpleExec(args, working_dir), SimpleExec.Result
)
if result.exit_code != 0:
recipe: Optional[Recipe] = current_recipe.get(None) # type: ignore
if recipe:
raise ExecutionFailure(
args,
working_dir,
recipe.get_host(),
recipe.get_user(kitchen.head),
result,
)
else:
raise ExecutionFailure(args, working_dir, "???", "???", result)
return result

7
scone/default/steps/filesystem_steps.py

@ -0,0 +1,7 @@
from scone.default.utensils.basic_utensils import HashFile
from scone.head.kitchen import Kitchen
async def depend_remote_file(path: str, kitchen: Kitchen) -> None:
sha256 = await kitchen.ut1(HashFile(path))
kitchen.get_dependency_tracker().register_remote_file(path, sha256)

74
scone/default/steps/fridge_steps.py

@ -0,0 +1,74 @@
from enum import Enum
from pathlib import Path, PurePath
from typing import List, Optional, Tuple, Union
from jinja2 import Template
from scone.head import Head
from scone.head.kitchen import Kitchen
SUPERMARKET_RELATIVE = ".scone-cache/supermarket"
def get_fridge_dirs(head: Head) -> List[Path]:
# TODO expand with per-sous/per-group dirs?
return [Path(head.directory, "fridge")]
def search_in_dirlist(
dirlist: List[Path], relative: Union[str, PurePath]
) -> Optional[Path]:
for directory in dirlist:
potential_path = directory.joinpath(relative)
if potential_path.exists():
return potential_path
return None
def search_in_fridge(head: Head, relative: Union[str, PurePath]) -> Optional[Path]:
fridge_dirs = get_fridge_dirs(head)
return search_in_dirlist(fridge_dirs, relative)
class FridgeMetadata(Enum):
FRIDGE = 0
FROZEN = 1
TEMPLATE = 2
def decode_fridge_extension(path: str) -> Tuple[str, FridgeMetadata]:
exts = {
".frozen": FridgeMetadata.FROZEN,