Accept workflows from the command line

This commit is contained in:
Benjamin Berman
2025-05-07 14:53:39 -07:00
parent da2cbf7c91
commit b6d3f1fb08
35 changed files with 459 additions and 319 deletions
+41
View File
@@ -0,0 +1,41 @@
import asyncio
try:
from collections.abc import Buffer
except ImportError:
from typing_extensions import Buffer
from io import BytesIO
from typing import Literal, AsyncGenerator
import ijson
import aiofiles
import sys
import shlex
async def stream_json_objects(source_path_or_stdin: str | Literal["-"]) -> AsyncGenerator[dict, None]:
"""
Asynchronously yields JSON objects from a given source.
The source can be a file path or "-" for stdin.
Assumes the input stream contains concatenated JSON objects (e.g., {}{}{}).
"""
if source_path_or_stdin is None or len(source_path_or_stdin) == 0:
return
elif source_path_or_stdin == "-":
async for obj in ijson.items_async(aiofiles.stdin_bytes, '', multiple_values=True):
yield obj
else:
# Handle file path or literal JSON
if "{" in source_path_or_stdin[:2]:
# literal string
encode: Buffer = source_path_or_stdin.encode("utf-8")
source_path_or_stdin = BytesIO(encode)
for obj in ijson.items(source_path_or_stdin, '', multiple_values=True):
yield obj
else:
async with aiofiles.open(source_path_or_stdin, mode='rb') as f:
# 'rb' mode is important as ijson expects byte streams.
# The prefix '' targets root-level objects.
# multiple_values=True allows parsing of multiple top-level JSON values.
async for obj in ijson.items_async(f, '', multiple_values=True):
yield obj
@@ -0,0 +1,41 @@
from typing import Optional
from ..cli_args_types import Configuration
from ..cmd.extra_model_paths import load_extra_path_config
from .folder_path_types import FolderNames
from ..component_model.platform_path import construct_path
import itertools
import os
from ..distributed.executors import ContextVarExecutor, ContextVarProcessPoolExecutor
def configure_application_paths(args: Configuration, folder_names: Optional[FolderNames] = None):
if folder_names is None:
from ..cmd import folder_paths
folder_names = folder_paths.folder_names_and_paths
# configure paths
if args.output_directory:
folder_names.application_paths.output_directory = construct_path(args.output_directory)
if args.input_directory:
folder_names.application_paths.input_directory = construct_path(args.input_directory)
if args.temp_directory:
folder_names.application_paths.temp_directory = construct_path(args.temp_directory)
if args.extra_model_paths_config:
for config_path in itertools.chain(*args.extra_model_paths_config):
load_extra_path_config(config_path, folder_names=folder_names)
async def executor_from_args(configuration:Optional[Configuration]=None):
if configuration is None:
from ..cli_args import args
configuration = args
if configuration.executor_factory in ("ThreadPoolExecutor", "ContextVarExecutor"):
executor = ContextVarExecutor()
elif configuration.executor_factory in ("ProcessPoolExecutor", "ContextVarProcessPoolExecutor"):
executor = ContextVarProcessPoolExecutor()
else:
# default executor
executor = ContextVarExecutor()
return executor
+14
View File
@@ -0,0 +1,14 @@
import contextlib
import io
import sys
@contextlib.contextmanager
def suppress_stdout_stderr():
new_stdout, new_stderr = io.StringIO(), io.StringIO()
old_stdout, old_stderr = sys.stdout, sys.stderr
try:
sys.stdout, sys.stderr = new_stdout, new_stderr
yield
finally:
sys.stdout, sys.stderr = old_stdout, old_stderr