Skip to content

Index

An optional SQLite index of a store, for questions across calculations.

The index is a cache: the store's files are the truth, and Index.rebuild remakes the index from them. Index.update instead reads the journal lines it has not seen and re-reads from the store every key they name, so updating and rebuilding end with the same index (a journal line only says which key changed; the rows always come from the store).

Tables:

  • runs: one row per run: key, run, time (UTC, ISO 8601), program, version (as the output reports it), calctype, method, basis, formula, success, error_kind, wall_time (s), identities (JSON), withdrawn, preferred (the run Store.get returns without identities), input (the input as JSON, files left out), and the environment it ran in, as its output reports it: host, cpu_model, devices (JSON list) and libc.
  • carries: key, run, carry (its id), identities (JSON), evidence (JSON), time: one row per run carried.
  • tags: key, tag.
  • meta: key, run, name, value (strings and numbers as they are; lists and objects as JSON).

Query JSON columns with SQLite's JSON functions: json_extract(identities, '$.tblite') = '0.8.0', json_extract(input, '$.keywords.maxiter').

qcstore.Index

Index(store: Store, path: Path)

A store's SQLite index; Store.index opens one, up to date.

Each method opens its own connection, so an index may be used from several threads and processes at once (on a local disk: SQLite's locks need one).

Attributes:

Name Type Description
store

The store indexed.

path

The SQLite file.

Source code in src/qcstore/index.py
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
def __init__(self, store: Store, path: Path) -> None:
    """Open the index file, creating or rebuilding it if needed, and update it."""
    self.store = store
    self.path = path
    path.parent.mkdir(parents=True, exist_ok=True)
    with self._writing() as db:
        fresh = db.execute("PRAGMA user_version").fetchone()[0] != SCHEMA
        if fresh:
            tables = db.execute("SELECT name FROM sqlite_master WHERE type='table'")
            for (name,) in tables.fetchall():
                db.execute(f"DROP TABLE {name}")
            for statement in _TABLES:
                db.execute(statement)
            db.execute(f"PRAGMA user_version = {SCHEMA}")
    if fresh:
        self.rebuild()
    else:
        self.update()

query

query(
    *,
    tag: str | None = None,
    identities: Mapping[str, str] | None = None,
    **columns: Any,
) -> list[dict[str, Any]]

Runs whose columns equal the values given, oldest first within each key.

Parameters:

Name Type Description Default
tag str | None

Only calculations with this tag.

None
identities Mapping[str, str] | None

Only runs valid under these numerics, produced by them or carried to them: {"tblite": "0.8.0"} matches every run whose tblite numerics is 0.8.0, whatever its other programs'.

None
**columns Any

Values of the runs columns (program="xtb", success=True, preferred=True); None matches NULL.

{}

Returns:

Type Description
list[dict[str, Any]]

One dict per run, with identities, input and devices parsed

list[dict[str, Any]]

from JSON.

Raises:

Type Description
ValueError

An unknown column, or a program name holding ".

Source code in src/qcstore/index.py
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def query(
    self,
    *,
    tag: str | None = None,
    identities: Mapping[str, str] | None = None,
    **columns: Any,
) -> list[dict[str, Any]]:
    """Runs whose columns equal the values given, oldest first within each key.

    Args:
        tag: Only calculations with this tag.
        identities: Only runs valid under these numerics, produced by them or
            carried to them: ``{"tblite": "0.8.0"}`` matches every run whose
            tblite numerics is 0.8.0, whatever its other programs'.
        **columns: Values of the ``runs`` columns (``program="xtb"``,
            ``success=True``, ``preferred=True``); None matches NULL.

    Returns:
        One dict per run, with ``identities``, ``input`` and ``devices`` parsed
        from JSON.

    Raises:
        ValueError: An unknown column, or a program name holding ``"``.
    """
    where: list[str] = []
    params: list[Any] = []
    for name, value in columns.items():
        if name not in _COLUMNS:
            raise ValueError(f"No column {name!r}; the columns are {_COLUMNS}.")
        where.append(f"{name} IS ?")
        params.append(value)
    if tag is not None:
        where.append("key IN (SELECT key FROM tags WHERE tag = ?)")
        params.append(tag)
    if identities:
        paths = []
        for program, numerics in identities.items():
            if '"' in program:
                raise ValueError(f"Not a program name: {program!r}.")
            paths += [f'$."{program}"', numerics]
        match = " AND ".join(
            ["json_extract({0}.identities, ?) = ?"] * len(identities)
        )
        where.append(
            f"(({match.format('runs')}) OR EXISTS (SELECT 1 FROM carries WHERE "
            f"carries.key = runs.key AND carries.run = runs.run AND "
            f"{match.format('carries')}))"
        )
        params += paths + paths
    text = "SELECT * FROM runs"
    if where:
        text += " WHERE " + " AND ".join(where)
    rows = self.sql(text + " ORDER BY key, run", params)
    for row in rows:
        for name in _BOOLEAN:
            row[name] = bool(row[name])
        for name in ("identities", "input", "devices"):
            row[name] = json.loads(row[name])
    return rows

rebuild

rebuild() -> None

Remake the index from every run in the store.

Source code in src/qcstore/index.py
130
131
132
133
134
135
136
137
138
139
140
141
142
def rebuild(self) -> None:
    """Remake the index from every run in the store."""
    with self._writing() as db:
        for table in (*_STORE_TABLES, "journals", "pending"):
            db.execute(f"DELETE FROM {table}")
        # Journals first: a change still running is pending, so the next update
        # re-reads its key even if this scan saw the key before it finished.
        offsets: dict[str, int] = {}
        pending: dict[tuple[str, int], str] = {}
        self._read_journals(offsets, pending)
        for key in self.store._keys():
            self._refresh(db, key)
        self._save_journals(db, offsets, pending)

sql

sql(
    text: str,
    params: Sequence[Any] | Mapping[str, Any] = (),
) -> list[dict[str, Any]]

Run a read-only SQL query; return one dict per row, by column name.

Example

How many calculations per molecule:

index.sql(
    "SELECT formula, COUNT(DISTINCT key) AS n FROM runs "
    "GROUP BY formula ORDER BY n DESC"
)
Source code in src/qcstore/index.py
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
def sql(
    self, text: str, params: Sequence[Any] | Mapping[str, Any] = ()
) -> list[dict[str, Any]]:
    """Run a read-only SQL query; return one dict per row, by column name.

    Example:
        How many calculations per molecule:

        ```python
        index.sql(
            "SELECT formula, COUNT(DISTINCT key) AS n FROM runs "
            "GROUP BY formula ORDER BY n DESC"
        )
        ```
    """
    uri = self.path.absolute().as_uri() + "?mode=ro"
    with closing(sqlite3.connect(uri, uri=True, timeout=60)) as db:
        cursor = db.execute(text, params)
        names = [d[0] for d in cursor.description or ()]
        return [dict(zip(names, row, strict=True)) for row in cursor.fetchall()]

update

update() -> None

Bring the index up to date: re-read every key the new journal lines name.

Source code in src/qcstore/index.py
118
119
120
121
122
123
124
125
126
127
128
def update(self) -> None:
    """Bring the index up to date: re-read every key the new journal lines name."""
    with self._writing() as db:
        offsets: dict[str, int] = dict(
            db.execute("SELECT path, offset FROM journals")
        )
        pending = {(p, n): k for p, n, k in db.execute("SELECT * FROM pending")}
        done = self._read_journals(offsets, pending)
        for key in sorted(done | set(pending.values())):
            self._refresh(db, key)
        self._save_journals(db, offsets, pending)