+
Skip to content

Create new gribscan CLI and use click to handle options #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 9, 2025
Merged

Create new gribscan CLI and use click to handle options #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 89 additions & 117 deletions gribscan/tools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import argparse
import glob
import json
import logging
Expand All @@ -7,136 +6,109 @@
from pathlib import Path
import multiprocessing as mp

import click

import gribscan
from .magician import MAGICIANS


def create_index():
parser = argparse.ArgumentParser()
parser.add_argument("sources", metavar="GRIB", help="source gribfile(s)", nargs="+")
parser.add_argument(
"-v",
"--verbose",
help="increase the logging level",
action="store_true",
)
parser.add_argument(
"-o",
"--outdir",
help="output directory to write index files",
type=str,
default=None,
nargs="?",
)
parser.add_argument(
"-f",
"--force",
help="overwrite existing index files",
action="store_true",
)
parser.add_argument(
"-n",
"--nprocs",
help="number of parallel processes",
type=int,
default=1,
nargs="?",
)
args = parser.parse_args()

@click.group()
@click.option("-v", "--verbose", is_flag=True, help="Increase the logging level.")
def cli(verbose):
"""gribscan: Index and build GRIB datasets."""
logging.basicConfig()
if args.verbose:
if verbose:
logging.getLogger().setLevel(logging.DEBUG)

mapfunc = partial(gribscan.write_index, outdir=args.outdir, force=args.force)
with mp.Pool(args.nprocs) as pool:
pool.map(mapfunc, args.sources)


def build_dataset():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument(
"-v",
"--verbose",
help="increase the logging level",
action="store_true",
)
parser.add_argument(
"indices",
metavar="GRIB.index",
help="source index files (JSONLines)",
nargs="*",
)
parser.add_argument(
"-g",
"--glob",
metavar="GLOB.index",
help="glob pattern to create list of index files (JSONLines)",
type=str,
nargs="?",
default=None,
)
parser.add_argument(
"-o",
"--output",
metavar="outdir",
default=".",
help="output directory to write reference filesystems (JSON)",
type=str,
)
parser.add_argument(
"--prefix",
metavar="template_prefix",
default=None,
help=textwrap.dedent("""\
Absolute path to the location of the dataset.

The prefix is prepended to the filename stored in the index files.
For full file paths, a sub-tree can be denoted using the '/./'
character. The following examples show how a prefix adds to
different filenames:
@cli.command("index")
@click.argument("sources", nargs=-1, type=click.Path(exists=True))
@click.option(
"-o",
"--outdir",
type=click.Path(file_okay=False, writable=True),
default=None,
help="Output directory to write index files.",
)
@click.option("-f", "--force", is_flag=True, help="Overwrite existing index files.")
@click.option(
"-n",
"--nprocs",
type=int,
default=1,
show_default=True,
help="Number of parallel processes.",
)
def create_index(sources, outdir, force, nprocs):
"""Create index files from GRIB sources."""
mapfunc = partial(gribscan.write_index, outdir=outdir, force=force)
with mp.Pool(nprocs) as pool:
pool.map(mapfunc, sources)


@cli.command("build")
@click.argument("indices", nargs=-1, type=click.Path(exists=True))
@click.option(
"-g",
"--glob",
"glob_pattern",
type=str,
help="Glob pattern to create list of index files (JSONLines).",
)
@click.option(
"-o",
"--output",
type=click.Path(file_okay=False, writable=True),
default=".",
show_default=True,
help="Output directory to write reference filesystems (JSON).",
)
@click.option(
"--prefix",
type=str,
help=textwrap.dedent(
"""\
Absolute path to the location of the dataset.
The prefix is prepended to the filename stored in the index files.
For full file paths, a sub-tree can be denoted using the '/./'
character. The following examples show how a prefix adds to
different filenames:
/prefix/ + filename.grb = /prefix/filename.grb
/prefix/ + path/filename.grb = /prefix/path/filename.grb
/prefix/ + path/./sub/tree/filename.grb = /prefix/sub/tree/filename.grb
"""
),
type=str,
)
parser.add_argument(
"-m",
"--magician",
metavar="magician",
default="monsoon",
help="use this magician for dataset assembly",
type=str,
)
args = parser.parse_args()

logging.basicConfig()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)

magician = MAGICIANS[args.magician]()

if args.glob is None and not args.indices:
parser.error("You need to pass a glob pattern or a file list.")

if args.glob is not None and args.indices:
parser.error("It is not possible to pass a glob pattern and a file list.")

if args.glob is not None:
indices = glob.iglob(args.glob)

if args.indices:
indices = args.indices

),
)
@click.option(
"-m",
"--magician",
default="monsoon",
show_default=True,
type=click.Choice(MAGICIANS.keys()),
help="Magician to use for dataset assembly.",
)
def build_dataset(indices, glob_pattern, output, prefix, magician):
"""Build dataset references from index files."""
if not glob_pattern and not indices:
raise click.UsageError("You must provide either a glob pattern or a file list.")
if glob_pattern and indices:
raise click.UsageError("Cannot provide both glob pattern and file list.")

if glob_pattern:
indices = list(glob.iglob(glob_pattern))

magician_instance = MAGICIANS[magician]()
refs = gribscan.grib_magic(
indices, magician=magician, global_prefix=args.prefix
indices, magician=magician_instance, global_prefix=prefix
)

Path(output).mkdir(parents=True, exist_ok=True)
for dataset, ref in refs.items():
with open(f"{args.output}/{dataset}.json", "w") as indexfile:
json.dump(ref, indexfile)
with open(Path(output) / f"{dataset}.json", "w") as f:
json.dump(ref, f, indent=2)


if __name__ == "__main__":
cli()
2 changes: 2 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ classifiers = [
]
dependencies = [
"cfgrib>=0.9.9.0", # previous versions create a cffi error on index
"click",
"eccodes",
"numcodecs>=0.10.0",
"numpy",
Expand All @@ -34,6 +35,7 @@ docs = [
]

[project.scripts]
gribscan = "gribscan.tools:cli"
gribscan-index = "gribscan.tools:create_index"
gribscan-build = "gribscan.tools:build_dataset"

Expand Down
点击 这是indexloc提供的php浏览器服务，不要输入任何密码和下载