diff --git a/browse/browse_test.go b/browse/browse_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6ac1793adf42ca2dffa34143b2bb9887ddfc0023 --- /dev/null +++ b/browse/browse_test.go @@ -0,0 +1,401 @@ +package browse + +import ( + "context" + "errors" + "testing" +) + +func TestBranches(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + branches, err := db.Branches(ctx) + if err != nil { + t.Fatal(err) + } + if len(branches) != 2 { + t.Fatalf("want 2 branches, got %d: %+v", len(branches), branches) + } + // main sorts first (default). + if branches[0].Name != "main" { + t.Errorf("want main first, got %q", branches[0].Name) + } + names := map[string]string{} + for _, b := range branches { + names[b.Name] = b.Head + } + if _, ok := names["dev"]; !ok { + t.Errorf("missing dev branch: %+v", branches) + } + // main head is C4 (add items); dev head is C2 (insert users): distinct. + if names["main"] == names["dev"] { + t.Errorf("main and dev unexpectedly share a head %q", names["main"]) + } + if got, want := names["main"], commitHash(t, msgAddItem); got != want { + t.Errorf("main head = %q, want C4 %q", got, want) + } + if got, want := names["dev"], commitHash(t, msgInsert); got != want { + t.Errorf("dev head = %q, want C2 %q", got, want) + } + + if got := DefaultBranch(branches); got != "main" { + t.Errorf("DefaultBranch = %q, want main", got) + } +} + +func TestLogTopoOrder(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + commits, next, err := db.Log(ctx, "main", "", 100) + if err != nil { + t.Fatal(err) + } + if next != "" { + t.Errorf("full log should have no next page, got %q", next) + } + wantOrder := []string{msgAddItem, msgModify, msgInsert, msgAddUser, msgInitial} + if len(commits) != len(wantOrder) { + t.Fatalf("want %d commits, got %d", len(wantOrder), len(commits)) + } + for i, want := range wantOrder { + if commits[i].Message != want { + t.Errorf("commit[%d] message = %q, want %q", i, commits[i].Message, want) + } + } + + // Author/email come from the CLI global config for the content commits. + if commits[0].Author != "CLI Fixture" || commits[0].Email != "cli@fixture.test" { + t.Errorf("head author/email = %q/%q", commits[0].Author, commits[0].Email) + } + + // Parent linkage: each commit's first parent is the next-older commit; + // the initial commit has no parent. + for i := 0; i < len(commits)-1; i++ { + if len(commits[i].ParentHashes) == 0 { + t.Fatalf("commit[%d] %q unexpectedly has no parent", i, commits[i].Message) + } + if commits[i].ParentHashes[0] != commits[i+1].Hash { + t.Errorf("commit[%d] parent = %q, want %q", i, commits[i].ParentHashes[0], commits[i+1].Hash) + } + } + if last := commits[len(commits)-1]; len(last.ParentHashes) != 0 { + t.Errorf("initial commit should have no parents, got %v", last.ParentHashes) + } + if commits[len(commits)-1].Date.IsZero() { + t.Errorf("initial commit date should be set") + } +} + +func TestLogPagination(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + // Page through the full history two commits at a time. + var got []string + from := "" + pages := 0 + for { + commits, next, err := db.Log(ctx, "main", from, 2) + if err != nil { + t.Fatal(err) + } + if len(commits) == 0 { + t.Fatalf("empty page (from=%q)", from) + } + if len(commits) > 2 { + t.Fatalf("page larger than limit: %d", len(commits)) + } + for _, c := range commits { + got = append(got, c.Message) + } + pages++ + if pages > 10 { + t.Fatal("pagination did not terminate") + } + if next == "" { + break + } + from = next + } + + want := []string{msgAddItem, msgModify, msgInsert, msgAddUser, msgInitial} + if len(got) != len(want) { + t.Fatalf("paged %d commits, want %d: %v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("paged[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +func TestLogFromHash(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + // Start the walk from C2 (insert users): should yield C2, C1, C0. + commits, next, err := db.Log(ctx, "main", commitHash(t, msgInsert), 100) + if err != nil { + t.Fatal(err) + } + if next != "" { + t.Errorf("want no next page, got %q", next) + } + want := []string{msgInsert, msgAddUser, msgInitial} + if len(commits) != len(want) { + t.Fatalf("want %d commits from C2, got %d", len(want), len(commits)) + } + for i, w := range want { + if commits[i].Message != w { + t.Errorf("from-hash commit[%d] = %q, want %q", i, commits[i].Message, w) + } + } +} + +func TestLogInvalidLimit(t *testing.T) { + db := openFixture(t) + if _, _, err := db.Log(context.Background(), "main", "", 0); err == nil { + t.Fatal("expected error for non-positive limit") + } +} + +func TestTables(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + tables, err := db.Tables(ctx, "main") + if err != nil { + t.Fatal(err) + } + byName := map[string]TableInfo{} + for _, ti := range tables { + byName[ti.Name] = ti + } + + users, ok := byName["users"] + if !ok { + t.Fatalf("missing users table: %+v", tables) + } + if users.RowCount != 4 { + t.Errorf("users row count = %d, want 4", users.RowCount) + } + if len(users.Columns) != 2 { + t.Fatalf("users columns = %d, want 2", len(users.Columns)) + } + id := users.Columns[0] + if id.Name != "id" || !id.PrimaryKey || id.Nullable { + t.Errorf("users.id column = %+v", id) + } + if id.Type != "int" { + t.Errorf("users.id type = %q, want int", id.Type) + } + name := users.Columns[1] + if name.Name != "name" || name.PrimaryKey || !name.Nullable { + t.Errorf("users.name column = %+v", name) + } + if name.Type != "varchar(64)" { + t.Errorf("users.name type = %q, want varchar(64)", name.Type) + } + + items, ok := byName["items"] + if !ok { + t.Fatalf("missing items table: %+v", tables) + } + if items.RowCount != 0 { + t.Errorf("items row count = %d, want 0", items.RowCount) + } +} + +func TestRowsPagination(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + first, err := db.Rows(ctx, "main", "users", 0, 2) + if err != nil { + t.Fatal(err) + } + if first.Total != 4 { + t.Errorf("total = %d, want 4", first.Total) + } + if first.Offset != 0 { + t.Errorf("offset = %d, want 0", first.Offset) + } + wantCols := []string{"id", "name"} + if len(first.Columns) != 2 || first.Columns[0] != wantCols[0] || first.Columns[1] != wantCols[1] { + t.Errorf("columns = %v, want %v", first.Columns, wantCols) + } + // Ordered by primary key: rows 1,2. id=1 was renamed to alicia in C3. + wantFirst := [][]string{{"1", "alicia"}, {"2", "bob"}} + if !equalRows(first.Rows, wantFirst) { + t.Errorf("page 1 rows = %v, want %v", first.Rows, wantFirst) + } + + second, err := db.Rows(ctx, "main", "users", 2, 2) + if err != nil { + t.Fatal(err) + } + wantSecond := [][]string{{"3", "carol"}, {"4", "dave"}} + if !equalRows(second.Rows, wantSecond) { + t.Errorf("page 2 rows = %v, want %v", second.Rows, wantSecond) + } + + // Offset at/after the end yields an empty page, not an error. + past, err := db.Rows(ctx, "main", "users", 4, 2) + if err != nil { + t.Fatal(err) + } + if len(past.Rows) != 0 { + t.Errorf("past-end page should be empty, got %v", past.Rows) + } + if past.Total != 4 { + t.Errorf("past-end total = %d, want 4", past.Total) + } +} + +func TestRowsEmptyTable(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + page, err := db.Rows(ctx, "main", "items", 0, 10) + if err != nil { + t.Fatal(err) + } + if page.Total != 0 { + t.Errorf("empty table total = %d, want 0", page.Total) + } + if len(page.Rows) != 0 { + t.Errorf("empty table rows = %v, want none", page.Rows) + } + // Columns are still reported so a UI can render an empty grid. + wantCols := []string{"sku", "qty"} + if len(page.Columns) != 2 || page.Columns[0] != wantCols[0] || page.Columns[1] != wantCols[1] { + t.Errorf("empty table columns = %v, want %v", page.Columns, wantCols) + } +} + +func TestCommitSummaryAddTable(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + // C1 creates the empty users table. + diff, err := db.CommitSummary(ctx, commitHash(t, msgAddUser)) + if err != nil { + t.Fatal(err) + } + td := findTable(t, diff, "users") + if !td.Added { + t.Errorf("users should be Added in C1: %+v", td) + } + if td.Dropped { + t.Errorf("users should not be Dropped: %+v", td) + } + if td.RowsAdded != 0 || td.RowsRemoved != 0 || td.RowsModified != 0 { + t.Errorf("C1 (empty table) row counts = %+v, want all 0", td) + } +} + +func TestCommitSummaryInsertRows(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + // C2 inserts 3 rows. + diff, err := db.CommitSummary(ctx, commitHash(t, msgInsert)) + if err != nil { + t.Fatal(err) + } + td := findTable(t, diff, "users") + if td.Added || td.Dropped { + t.Errorf("C2 users should be a data change, not add/drop: %+v", td) + } + if td.RowsAdded != 3 { + t.Errorf("C2 RowsAdded = %d, want 3", td.RowsAdded) + } + if td.RowsRemoved != 0 || td.RowsModified != 0 { + t.Errorf("C2 unexpected removed/modified: %+v", td) + } +} + +func TestCommitSummaryModifyRows(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + // C3 updates id=1 and inserts id=4. + diff, err := db.CommitSummary(ctx, commitHash(t, msgModify)) + if err != nil { + t.Fatal(err) + } + td := findTable(t, diff, "users") + if td.RowsAdded != 1 { + t.Errorf("C3 RowsAdded = %d, want 1", td.RowsAdded) + } + if td.RowsModified != 1 { + t.Errorf("C3 RowsModified = %d, want 1", td.RowsModified) + } + if td.RowsRemoved != 0 { + t.Errorf("C3 RowsRemoved = %d, want 0", td.RowsRemoved) + } +} + +func TestCommitSummaryInitial(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + // C0 is the initial (parentless) commit, diffed against the empty root. + // WriteEmptyRepo's initial commit has no tables, so the summary is empty + // but must not error and must exercise the empty-root path. + diff, err := db.CommitSummary(ctx, commitHash(t, msgInitial)) + if err != nil { + t.Fatalf("initial commit summary: %v", err) + } + if len(diff.Tables) != 0 { + t.Errorf("initial commit should have no table changes, got %+v", diff.Tables) + } + if diff.Hash != commitHash(t, msgInitial) { + t.Errorf("summary hash = %q, want %q", diff.Hash, commitHash(t, msgInitial)) + } +} + +func TestErrorsNonexistentRefAndTable(t *testing.T) { + db := openFixture(t) + ctx := context.Background() + + if _, err := db.Tables(ctx, "no-such-ref"); !errors.Is(err, ErrRefNotFound) { + t.Errorf("Tables(bad ref) err = %v, want ErrRefNotFound", err) + } + if _, _, err := db.Log(ctx, "no-such-ref", "", 10); !errors.Is(err, ErrRefNotFound) { + t.Errorf("Log(bad ref) err = %v, want ErrRefNotFound", err) + } + if _, err := db.Rows(ctx, "main", "no-such-table", 0, 10); !errors.Is(err, ErrTableNotFound) { + t.Errorf("Rows(bad table) err = %v, want ErrTableNotFound", err) + } +} + +func findTable(t *testing.T, diff *CommitDiff, name string) TableDiff { + t.Helper() + for _, td := range diff.Tables { + if td.Name == name { + return td + } + } + t.Fatalf("table %q not in diff %+v", name, diff.Tables) + return TableDiff{} +} + +func equalRows(got, want [][]string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if len(got[i]) != len(want[i]) { + return false + } + for j := range got[i] { + if got[i][j] != want[i][j] { + return false + } + } + } + return true +} diff --git a/browse/diff.go b/browse/diff.go new file mode 100644 index 0000000000000000000000000000000000000000..3ac076430e530ddf0404da17290ad7f036faa9b5 --- /dev/null +++ b/browse/diff.go @@ -0,0 +1,168 @@ +package browse + +import ( + "context" + "errors" + "fmt" + + "github.com/dolthub/dolt/go/libraries/doltcore/diff" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" +) + +// TableDiff summarizes how one table changed in a commit relative to its first +// parent. Row-level diffs are out of scope for v1; only the per-table shape and +// exact row counts are reported. +type TableDiff struct { + Name string + Added bool + Dropped bool + SchemaChanged bool + // RowsAdded, RowsRemoved and RowsModified are exact counts derived from the + // prolly diff. They are 0 when counts could not be computed (e.g. the + // primary key set changed, in which case SchemaChanged is true). + RowsAdded int64 + RowsRemoved int64 + RowsModified int64 +} + +// CommitDiff is the per-table summary of a single commit versus its first +// parent (or the empty root, for the initial commit). +type CommitDiff struct { + Hash string + Tables []TableDiff +} + +// CommitSummary computes the per-table diff summary of the commit identified by +// hashStr (a commit hash, or any ref resolveCommit accepts) against its first +// parent. For the initial commit the comparison is against an empty root, so +// every table shows as added. +func (db *DB) CommitSummary(ctx context.Context, hashStr string) (*CommitDiff, error) { + commit, err := db.resolveCommit(ctx, hashStr) + if err != nil { + return nil, err + } + + toRoot, err := commit.GetRootValue(ctx) + if err != nil { + return nil, fmt.Errorf("browse: root value of %q: %w", hashStr, err) + } + + fromRoot, err := db.firstParentRoot(ctx, commit) + if err != nil { + return nil, err + } + + deltas, err := diff.GetTableDeltas(ctx, fromRoot, toRoot) + if err != nil { + return nil, fmt.Errorf("browse: table deltas for %q: %w", hashStr, err) + } + + h, err := commit.HashOf() + if err != nil { + return nil, fmt.Errorf("browse: hash of %q: %w", hashStr, err) + } + + out := &CommitDiff{Hash: h.String(), Tables: make([]TableDiff, 0, len(deltas))} + for _, delta := range deltas { + td, err := tableDiff(ctx, delta) + if err != nil { + return nil, err + } + out.Tables = append(out.Tables, td) + } + + return out, nil +} + +// firstParentRoot returns the root value of the commit's first parent, or an +// empty root when the commit has no parents (the initial commit). +func (db *DB) firstParentRoot(ctx context.Context, commit *doltdb.Commit) (doltdb.RootValue, error) { + parents, err := commit.ParentHashes(ctx) + if err != nil { + return nil, fmt.Errorf("browse: parent hashes: %w", err) + } + + if len(parents) == 0 { + root, err := doltdb.EmptyRootValue(ctx, db.ddb.ValueReadWriter(), db.ddb.NodeStore()) + if err != nil { + return nil, fmt.Errorf("browse: empty root: %w", err) + } + return root, nil + } + + oc, err := db.ddb.ResolveHash(ctx, parents[0]) + if err != nil { + return nil, fmt.Errorf("browse: resolve parent %s: %w", parents[0].String(), err) + } + parent, ok := oc.ToCommit() + if !ok { + return nil, fmt.Errorf("browse: parent commit %s is not resolvable (ghost)", parents[0].String()) + } + root, err := parent.GetRootValue(ctx) + if err != nil { + return nil, fmt.Errorf("browse: parent root %s: %w", parents[0].String(), err) + } + return root, nil +} + +// tableDiff summarizes a single TableDelta. +func tableDiff(ctx context.Context, delta diff.TableDelta) (TableDiff, error) { + name := delta.ToName.Name + if delta.IsDrop() { + name = delta.FromName.Name + } + + td := TableDiff{ + Name: name, + Added: delta.IsAdd(), + Dropped: delta.IsDrop(), + } + + schemaChanged, err := delta.HasSchemaChanged(ctx) + if err != nil { + return TableDiff{}, fmt.Errorf("browse: schema-changed check for %q: %w", name, err) + } + td.SchemaChanged = schemaChanged + + added, removed, modified, err := tableRowStat(ctx, delta) + if err != nil { + if errors.Is(err, diff.ErrPrimaryKeySetChanged) { + // Row counts are undefined when the PK set changed; the schema + // change is already reflected. Leave counts at 0. + return td, nil + } + return TableDiff{}, fmt.Errorf("browse: row stat for %q: %w", name, err) + } + td.RowsAdded = added + td.RowsRemoved = removed + td.RowsModified = modified + + return td, nil +} + +// tableRowStat drains diff.StatForTableDelta into exact added/removed/modified +// row counts. StatForTableDelta streams one progress message per changed row, +// so we accumulate the per-message Adds/Removes/Changes. +func tableRowStat(ctx context.Context, delta diff.TableDelta) (added, removed, modified int64, err error) { + ch := make(chan diff.DiffStatProgress, 128) + done := make(chan struct{}) + var statErr error + go func() { + defer close(done) + statErr = diff.StatForTableDelta(ctx, ch, delta) + close(ch) + }() + + var a, r, m uint64 + for p := range ch { + a += p.Adds + r += p.Removes + m += p.Changes + } + <-done + + if statErr != nil { + return 0, 0, 0, statErr + } + return int64(a), int64(r), int64(m), nil +} diff --git a/browse/fixture_test.go b/browse/fixture_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ff2ce9fcbca40a2ff3ec607fc0170438c59e778c --- /dev/null +++ b/browse/fixture_test.go @@ -0,0 +1,207 @@ +package browse + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/utils/earl" + "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/types" +) + +// The browse tests read a real bare NBS store built the way production stores +// are: doltdb.WriteEmptyRepo creates the bare remote, then a working clone is +// grown with the dolt CLI and pushed back over a file:// remote. That push +// path produces exactly our production bare-store shape, and building the +// fixture through the CLI keeps this package's tests independent of the +// version-fragile editor/prolly write APIs (the same fragility this package is +// meant to contain on the read side). Programmatic table building via the +// doltdb editor APIs was considered and rejected as disproportionately +// complex and brittle for a read-only browse layer. +// +// The fixture is built once in TestMain and shared read-only across tests. +// When the dolt CLI is absent the tests skip rather than fail. + +const doltBin = "/opt/homebrew/bin/dolt" + +// Commit messages, oldest to newest on main. +const ( + msgInitial = "Initialize data repository" // C0, from WriteEmptyRepo, no parent + msgAddUser = "add users" // C1: create empty users table + msgInsert = "insert users" // C2: insert 3 rows + msgModify = "modify users" // C3: update 1 row, insert 1 row + msgAddItem = "add items" // C4: create empty items table +) + +var ( + haveDolt bool + fixtureStore string // path to the bare store, main head = C4 + hashByMessage = map[string]string{} +) + +func TestMain(m *testing.M) { + os.Exit(runWithFixture(m)) +} + +// runWithFixture builds the fixture (if dolt is available), runs the tests, and +// cleans up. Split out so its defer runs before os.Exit. +func runWithFixture(m *testing.M) int { + if _, err := os.Stat(doltBin); err != nil { + haveDolt = false + return m.Run() + } + haveDolt = true + + dir, err := os.MkdirTemp("", "browse-fixture-") + if err != nil { + fmt.Fprintln(os.Stderr, "browse fixture: mkdtemp:", err) + return 1 + } + defer os.RemoveAll(dir) + + store, err := buildFixture(dir) + if err != nil { + fmt.Fprintln(os.Stderr, "browse fixture build failed:", err) + return 1 + } + fixtureStore = store + + if err := indexCommits(store); err != nil { + fmt.Fprintln(os.Stderr, "browse fixture index failed:", err) + return 1 + } + + return m.Run() +} + +// buildFixture creates the bare store and grows it via the CLI, returning the +// bare store path. +func buildFixture(root string) (string, error) { + ctx := context.Background() + + bare := filepath.Join(root, "bare") + if err := os.MkdirAll(bare, 0o755); err != nil { + return "", err + } + url := earl.FileUrlFromPath(bare, os.PathSeparator) + ddb, err := doltdb.LoadDoltDB(ctx, types.Format_DOLT, url, filesys.LocalFS) + if err != nil { + return "", fmt.Errorf("load bare: %w", err) + } + if err := ddb.WriteEmptyRepo(ctx, "main", "Fixture Owner", "owner@fixture.test"); err != nil { + return "", fmt.Errorf("write empty repo: %w", err) + } + // Safe to close immediately after WriteEmptyRepo (novel tables only); the + // generational-close panic documented in open.go only bites on reopen. + if err := ddb.Close(); err != nil { + return "", fmt.Errorf("close bare: %w", err) + } + + home := filepath.Join(root, "home") + if err := os.MkdirAll(home, 0o755); err != nil { + return "", err + } + env := append(os.Environ(), "HOME="+home) + + run := func(dir string, args ...string) error { + cmd := exec.Command(doltBin, args...) + cmd.Dir = dir + cmd.Env = env + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("dolt %s: %w\n%s", strings.Join(args, " "), err, out) + } + return nil + } + + steps := [][]string{ + {"config", "--global", "--add", "user.email", "cli@fixture.test"}, + {"config", "--global", "--add", "user.name", "CLI Fixture"}, + } + for _, s := range steps { + if err := run(root, s...); err != nil { + return "", err + } + } + + work := filepath.Join(root, "work") + if err := run(root, "clone", "file://"+filepath.ToSlash(bare), work); err != nil { + return "", err + } + + workSteps := [][]string{ + {"sql", "-q", "create table users (id int primary key, name varchar(64))"}, + {"commit", "-Am", msgAddUser}, + {"sql", "-q", "insert into users values (1,'alice'),(2,'bob'),(3,'carol')"}, + {"commit", "-Am", msgInsert}, + {"branch", "dev"}, // dev head = C2 + {"sql", "-q", "update users set name='alicia' where id=1; insert into users values (4,'dave')"}, + {"commit", "-Am", msgModify}, + {"sql", "-q", "create table items (sku varchar(16) primary key, qty int)"}, + {"commit", "-Am", msgAddItem}, + {"push", "origin", "main"}, + {"push", "origin", "dev"}, + } + for _, s := range workSteps { + if err := run(work, s...); err != nil { + return "", err + } + } + + return bare, nil +} + +// indexCommits records the hash of each commit on main by its message. +func indexCommits(store string) error { + ctx := context.Background() + db, err := Open(ctx, store) + if err != nil { + return err + } + defer db.Close() + + commits, _, err := db.Log(ctx, "main", "", 100) + if err != nil { + return err + } + for _, c := range commits { + hashByMessage[c.Message] = c.Hash + } + return nil +} + +// requireFixture skips the test when the dolt CLI (and thus the fixture) is +// unavailable. +func requireFixture(t *testing.T) { + t.Helper() + if !haveDolt { + t.Skipf("dolt CLI not found at %s; skipping browse tests", doltBin) + } +} + +// openFixture opens the shared fixture store for one test. +func openFixture(t *testing.T) *DB { + t.Helper() + requireFixture(t) + db, err := Open(context.Background(), fixtureStore) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +// commitHash returns the recorded hash for a commit message. +func commitHash(t *testing.T, message string) string { + t.Helper() + h, ok := hashByMessage[message] + if !ok { + t.Fatalf("no recorded commit for message %q", message) + } + return h +} diff --git a/browse/log.go b/browse/log.go new file mode 100644 index 0000000000000000000000000000000000000000..4ba16a056f6e7d1706b81dbda972b1cf347e4d5b --- /dev/null +++ b/browse/log.go @@ -0,0 +1,214 @@ +package browse + +import ( + "context" + "errors" + "fmt" + "io" + "sort" + "time" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/env/actions/commitwalk" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" + "github.com/dolthub/dolt/go/store/datas" + "github.com/dolthub/dolt/go/store/hash" +) + +// defaultBranchName is preferred as the default branch when present. +const defaultBranchName = "main" + +// Branch is a named branch and the hash of its head commit. +type Branch struct { + Name string + Head string +} + +// CommitInfo is a single commit in a log listing. +type CommitInfo struct { + Hash string + Author string + Email string + Date time.Time + Message string + ParentHashes []string +} + +// Branches returns all branches in the store, sorted by name with the default +// branch (see DefaultBranch) first. +func (db *DB) Branches(ctx context.Context) ([]Branch, error) { + refs, err := db.ddb.GetBranchesWithHashes(ctx) + if err != nil { + return nil, fmt.Errorf("browse: list branches: %w", err) + } + + branches := make([]Branch, 0, len(refs)) + for _, r := range refs { + branches = append(branches, Branch{Name: r.Ref.GetPath(), Head: r.Hash.String()}) + } + + sort.Slice(branches, func(i, j int) bool { + // "main" sorts before everything else; otherwise alphabetical. + if branches[i].Name == defaultBranchName { + return branches[j].Name != defaultBranchName + } + if branches[j].Name == defaultBranchName { + return false + } + return branches[i].Name < branches[j].Name + }) + + return branches, nil +} + +// DefaultBranch picks the default branch name from a list produced by +// Branches: "main" if present, otherwise the first branch. It returns "" when +// the list is empty. +func DefaultBranch(branches []Branch) string { + for _, b := range branches { + if b.Name == defaultBranchName { + return b.Name + } + } + if len(branches) > 0 { + return branches[0].Name + } + return "" +} + +// Log returns up to limit commits in reverse-topological order starting from +// the head of ref (a branch name or a commit hash). When fromHash is non-empty +// the walk starts there instead of ref's head, which is how pages after the +// first are fetched: pass the nextHash returned by the previous call. nextHash +// is the hash of the first commit of the following page, or "" when the last +// page was returned. +func (db *DB) Log(ctx context.Context, refStr, fromHash string, limit int) ([]CommitInfo, string, error) { + if limit <= 0 { + return nil, "", fmt.Errorf("browse: log limit must be positive, got %d", limit) + } + + var start hash.Hash + if fromHash != "" { + h, ok := hash.MaybeParse(fromHash) + if !ok { + return nil, "", fmt.Errorf("browse: invalid from hash %q", fromHash) + } + start = h + } else { + c, err := db.resolveCommit(ctx, refStr) + if err != nil { + return nil, "", err + } + start, err = c.HashOf() + if err != nil { + return nil, "", fmt.Errorf("browse: head hash of %q: %w", refStr, err) + } + } + + itr, err := commitwalk.GetTopologicalOrderIterator[context.Context](ctx, db.ddb, []hash.Hash{start}, nil) + if err != nil { + return nil, "", fmt.Errorf("browse: topological iterator: %w", err) + } + + out := make([]CommitInfo, 0, limit) + nextHash := "" + for { + h, oc, meta, _, err := itr.Next(ctx) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, "", fmt.Errorf("browse: walk commits: %w", err) + } + + if len(out) == limit { + // One past the requested page: its hash is the next page's start. + nextHash = h.String() + break + } + + ci, err := commitInfo(ctx, h, oc, meta) + if err != nil { + return nil, "", err + } + out = append(out, ci) + } + + return out, nextHash, nil +} + +// commitInfo builds a CommitInfo from the iterator's outputs. meta may be +// supplied by the iterator; when nil we read it from the commit. +func commitInfo(ctx context.Context, h hash.Hash, oc *doltdb.OptionalCommit, meta *datas.CommitMeta) (CommitInfo, error) { + commit, ok := oc.ToCommit() + if !ok { + return CommitInfo{}, fmt.Errorf("browse: commit %s is not resolvable (ghost)", h.String()) + } + + if meta == nil { + var err error + meta, err = commit.GetCommitMeta(ctx) + if err != nil { + return CommitInfo{}, fmt.Errorf("browse: commit meta %s: %w", h.String(), err) + } + } + + parents, err := commit.ParentHashes(ctx) + if err != nil { + return CommitInfo{}, fmt.Errorf("browse: parent hashes %s: %w", h.String(), err) + } + parentStrs := make([]string, len(parents)) + for i, p := range parents { + parentStrs[i] = p.String() + } + + return CommitInfo{ + Hash: h.String(), + Author: meta.Author.Name, + Email: meta.Author.Email, + Date: time.UnixMilli(int64(meta.TimestampMillis())), + Message: meta.Description, + ParentHashes: parentStrs, + }, nil +} + +// resolveCommit resolves a ref string to a commit. It is tried first as a +// branch name, then as a commit hash. +func (db *DB) resolveCommit(ctx context.Context, refStr string) (*doltdb.Commit, error) { + if _, ok, err := db.ddb.HasBranch(ctx, refStr); err != nil { + return nil, fmt.Errorf("browse: check branch %q: %w", refStr, err) + } else if ok { + c, err := db.ddb.ResolveCommitRef(ctx, ref.NewBranchRef(refStr)) + if err != nil { + return nil, fmt.Errorf("browse: resolve branch %q: %w", refStr, err) + } + return c, nil + } + + if h, ok := hash.MaybeParse(refStr); ok { + oc, err := db.ddb.ResolveHash(ctx, h) + if err != nil { + return nil, fmt.Errorf("browse: resolve hash %q: %w", refStr, err) + } + c, ok := oc.ToCommit() + if !ok { + return nil, fmt.Errorf("%w: %s", ErrRefNotFound, refStr) + } + return c, nil + } + + return nil, fmt.Errorf("%w: %s", ErrRefNotFound, refStr) +} + +// resolveRoot resolves a ref string to its committed root value. +func (db *DB) resolveRoot(ctx context.Context, refStr string) (doltdb.RootValue, error) { + c, err := db.resolveCommit(ctx, refStr) + if err != nil { + return nil, err + } + root, err := c.GetRootValue(ctx) + if err != nil { + return nil, fmt.Errorf("browse: root value for %q: %w", refStr, err) + } + return root, nil +} diff --git a/browse/open.go b/browse/open.go new file mode 100644 index 0000000000000000000000000000000000000000..00d6df49869bb226c2e30f5148fb8c0e09e6f3ba --- /dev/null +++ b/browse/open.go @@ -0,0 +1,109 @@ +// Package browse is the read-only web-browsing data layer over bare Dolt +// chunk stores. It is deliberately the only package in dolt.sr.ht that reaches +// into low-level dolthub/dolt internals (nbs, prolly, durable, diff), so that +// all version-fragile code stays contained here and a future module bump only +// needs to be re-verified against this one package. +// +// # Bare stores have no working set +// +// dolt.sr.ht serves bare NBS chunk-store directories (created via +// doltdb.WriteEmptyRepo and grown by pushes over the remotesapi). They have no +// .dolt/ working set, so the sqle engine and the embedded driver cannot open +// them. Everything here reads from committed roots only. +// +// # Open discipline (why not doltdb.LoadDoltDB) +// +// The obvious entry point, doltdb.LoadDoltDB, routes through the file +// dbfactory, which wraps the store in a GenerationalNBS (newgen + an oldgen +// subdir + a ghost gen). In the pinned dolt/go version +// (v0.40.5-0.20260626152440-45335d44ad79), calling Close() on such a +// generational store over one of our bare stores panics deep in nbs: +// +// panic: Close() called and reduced ref count to < 0. +// store/nbs/table_index.go:532 onHeapTableIndex.Close +// ... GenerationalNBS.Close -> NomsBlockStore.Close -> tableSet.close +// +// It reproduces on a plain open-read-only-then-Close, independent of any +// reads, so the documented open-per-request + Close pattern would crash the +// web process, not just tests. A single (non-generational) nbs.NewLocalStore +// — the exact construction storage.Cache uses to serve pushes — closes +// cleanly. We therefore build the DoltDB by hand from one NewLocalStore via +// doltdb.DoltDBFromCS and never touch the generational path. +// +// # Freshness and concurrency +// +// Open is called once per request and Close releases the handle, so every +// request gets a fresh read of the on-disk manifest and observes commits +// landed by the push writer since the last request — no stale cache, no +// refresh dance. NBS readers are manifest-based and append-only: opening a +// second NewLocalStore over a directory the push writer is also serving is +// safe (no exclusive lock is held for the store's lifetime; the manifest lock +// is taken only briefly during atomic updates), so a reader always sees a +// consistent snapshot. +package browse + +import ( + "context" + "errors" + "fmt" + "path/filepath" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/store/nbs" + "github.com/dolthub/dolt/go/store/types" +) + +// readStoreMemTableSize bounds the in-memory memtable of the read handle. It +// matches the value storage.Cache uses for served stores; browsing never +// writes, so the memtable stays empty in practice. +const readStoreMemTableSize = 128 * 1024 * 1024 + +// ErrRefNotFound is returned when a ref string matches neither an existing +// branch nor a resolvable commit hash. +var ErrRefNotFound = errors.New("browse: ref not found") + +// ErrTableNotFound is returned when a table does not exist in the resolved +// root value. +var ErrTableNotFound = errors.New("browse: table not found") + +// DB is a read-only handle to a single bare Dolt chunk store. It is not safe +// for concurrent use; open one per request and Close it when done. +type DB struct { + ddb *doltdb.DoltDB + path string +} + +// Open opens the bare Dolt chunk store at diskPath for read-only browsing. +// diskPath is the store directory itself (the dir that holds the NBS manifest +// and table files), not a parent. The caller must Close the returned DB. +// +// See the package doc for why this bypasses doltdb.LoadDoltDB. +func Open(ctx context.Context, diskPath string) (*DB, error) { + cs, err := nbs.NewLocalStore( + ctx, + types.Format_DOLT.VersionString(), + diskPath, + readStoreMemTableSize, + nbs.NewUnlimitedMemQuotaProvider(), + false, + ) + if err != nil { + return nil, fmt.Errorf("browse: open store %q: %w", diskPath, err) + } + + ddb, err := doltdb.DoltDBFromCS(cs, filepath.Base(diskPath)) + if err != nil { + // cs is not yet owned by a DoltDB, close it directly. + _ = cs.Close() + return nil, fmt.Errorf("browse: build doltdb for %q: %w", diskPath, err) + } + + return &DB{ddb: ddb, path: diskPath}, nil +} + +// Close releases the underlying chunk store. It closes the single +// non-generational NBS store, which is safe in the pinned version (unlike the +// generational store LoadDoltDB would have produced). +func (db *DB) Close() error { + return db.ddb.Close() +} diff --git a/browse/tables.go b/browse/tables.go new file mode 100644 index 0000000000000000000000000000000000000000..7c08df9f8f29cd063de994a270580018b0e04d8e --- /dev/null +++ b/browse/tables.go @@ -0,0 +1,279 @@ +package browse + +import ( + "context" + "fmt" + "io" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/store/val" +) + +// Cell placeholders. Rendering must never panic and never emit non-printable +// bytes; exotic or out-of-band values degrade to one of these. +const ( + placeholderNull = "NULL" + placeholderBinary = "" + placeholderUnreadable = "" +) + +// ColumnInfo describes one column of a table schema. +type ColumnInfo struct { + Name string + Type string + PrimaryKey bool + Nullable bool +} + +// TableInfo is a table name, its schema, and its row count at a ref. +type TableInfo struct { + Name string + Columns []ColumnInfo + RowCount uint64 +} + +// RowPage is a paginated slice of a table's rows rendered to strings. Columns +// lists the column names in the same order as each row's cells. For keyed +// tables columns are primary-key columns first, then the rest. +type RowPage struct { + Columns []string + Rows [][]string + Offset int + Total int +} + +// Tables lists the tables in the committed root at ref (a branch name or +// commit hash), each with its schema and row count. +func (db *DB) Tables(ctx context.Context, refStr string) ([]TableInfo, error) { + root, err := db.resolveRoot(ctx, refStr) + if err != nil { + return nil, err + } + + names, err := root.GetTableNames(ctx, doltdb.DefaultSchemaName, false) + if err != nil { + return nil, fmt.Errorf("browse: list tables at %q: %w", refStr, err) + } + + infos := make([]TableInfo, 0, len(names)) + for _, name := range names { + tbl, ok, err := root.GetTable(ctx, doltdb.TableName{Name: name}) + if err != nil { + return nil, fmt.Errorf("browse: get table %q: %w", name, err) + } + if !ok { + // Listed by GetTableNames but not resolvable: inconsistent root. + return nil, fmt.Errorf("browse: table %q listed but missing", name) + } + + sch, err := tbl.GetSchema(ctx) + if err != nil { + return nil, fmt.Errorf("browse: schema of %q: %w", name, err) + } + + idx, err := tbl.GetRowData(ctx) + if err != nil { + return nil, fmt.Errorf("browse: row data of %q: %w", name, err) + } + count, err := idx.Count() + if err != nil { + return nil, fmt.Errorf("browse: row count of %q: %w", name, err) + } + + infos = append(infos, TableInfo{ + Name: name, + Columns: columnInfos(sch), + RowCount: count, + }) + } + + return infos, nil +} + +// columnInfos renders a schema's columns in natural table order. +func columnInfos(sch schema.Schema) []ColumnInfo { + cols := sch.GetAllCols().GetColumns() + out := make([]ColumnInfo, len(cols)) + for i, c := range cols { + typeStr := "" + if c.TypeInfo != nil { + if sqlType := c.TypeInfo.ToSqlType(); sqlType != nil { + typeStr = sqlType.String() + } else { + typeStr = c.TypeInfo.String() + } + } + out[i] = ColumnInfo{ + Name: c.Name, + Type: typeStr, + PrimaryKey: c.IsPartOfPK, + Nullable: c.IsNullable(), + } + } + return out +} + +// cellRef locates a column's value within a prolly row: either in the key +// tuple or the value tuple, at the given field index. +type cellRef struct { + fromKey bool + idx int +} + +// Rows returns a page of rows from table at ref, starting at offset (0-based) +// and returning at most limit rows. The page is read directly from the prolly +// map via an ordinal range, so it is O(limit) regardless of offset. +func (db *DB) Rows(ctx context.Context, refStr, table string, offset, limit int) (*RowPage, error) { + if offset < 0 { + return nil, fmt.Errorf("browse: offset must be non-negative, got %d", offset) + } + if limit <= 0 { + return nil, fmt.Errorf("browse: limit must be positive, got %d", limit) + } + + root, err := db.resolveRoot(ctx, refStr) + if err != nil { + return nil, err + } + + tbl, ok, err := root.GetTable(ctx, doltdb.TableName{Name: table}) + if err != nil { + return nil, fmt.Errorf("browse: get table %q: %w", table, err) + } + if !ok { + return nil, fmt.Errorf("%w: %s", ErrTableNotFound, table) + } + + sch, err := tbl.GetSchema(ctx) + if err != nil { + return nil, fmt.Errorf("browse: schema of %q: %w", table, err) + } + + idx, err := tbl.GetRowData(ctx) + if err != nil { + return nil, fmt.Errorf("browse: row data of %q: %w", table, err) + } + total, err := idx.Count() + if err != nil { + return nil, fmt.Errorf("browse: row count of %q: %w", table, err) + } + + colNames, refs := rowLayout(sch) + + page := &RowPage{ + Columns: colNames, + Rows: [][]string{}, + Offset: offset, + Total: int(total), + } + + start := uint64(offset) + if start >= total { + // Past the end (also covers the empty-table case): no rows. + return page, nil + } + stop := start + uint64(limit) + if stop > total { + stop = total + } + + m, err := durable.ProllyMapFromIndex(idx) + if err != nil { + return nil, fmt.Errorf("browse: prolly map of %q: %w", table, err) + } + keyDesc, valDesc := m.Descriptors() + + iter, err := m.IterOrdinalRange(ctx, start, stop) + if err != nil { + return nil, fmt.Errorf("browse: iterate rows of %q: %w", table, err) + } + + for { + key, value, err := iter.Next(ctx) + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("browse: read row of %q: %w", table, err) + } + + row := make([]string, len(refs)) + for i, r := range refs { + if r.fromKey { + row[i] = renderCell(ctx, keyDesc, r.idx, key) + } else { + row[i] = renderCell(ctx, valDesc, r.idx, value) + } + } + page.Rows = append(page.Rows, row) + } + + return page, nil +} + +// rowLayout maps schema columns to their position in the prolly key/value +// tuples and produces the display column order. +// +// - Keyed tables: primary-key columns (in key order) map to the key tuple, +// the remaining columns (in stored order) to the value tuple. +// - Keyless tables: every column is in the value tuple; field 0 is the +// hidden cardinality, so column i lives at value index i+1, and there is +// no meaningful key. +func rowLayout(sch schema.Schema) ([]string, []cellRef) { + if schema.IsKeyless(sch) { + cols := sch.GetNonPKCols().GetColumns() + names := make([]string, len(cols)) + refs := make([]cellRef, len(cols)) + for i, c := range cols { + names[i] = c.Name + refs[i] = cellRef{fromKey: false, idx: i + 1} + } + return names, refs + } + + pkCols := sch.GetPKCols().GetColumns() + nonPKCols := sch.GetNonPKCols().GetColumns() + names := make([]string, 0, len(pkCols)+len(nonPKCols)) + refs := make([]cellRef, 0, len(pkCols)+len(nonPKCols)) + for i, c := range pkCols { + names = append(names, c.Name) + refs = append(refs, cellRef{fromKey: true, idx: i}) + } + for i, c := range nonPKCols { + names = append(names, c.Name) + refs = append(refs, cellRef{fromKey: false, idx: i}) + } + return names, refs +} + +// renderCell renders one tuple field to a display string. It never panics +// (recovering into a placeholder) and maps binary / out-of-band encodings to +// "" so a row preview never dumps opaque bytes. +func renderCell(ctx context.Context, td *val.TupleDesc, i int, tup val.Tuple) (out string) { + defer func() { + if r := recover(); r != nil { + out = placeholderUnreadable + } + }() + + if i < 0 || i >= td.Count() { + // Column not present in this tuple (e.g. a virtual/dropped column that + // isn't materialized): degrade rather than index out of range. + return placeholderUnreadable + } + if td.IsNull(i, tup) { + return placeholderNull + } + + switch td.Types[i].Enc { + case val.ByteStringEnc, val.Hash128Enc, val.CellEnc, + val.BytesAddrEnc, val.StringAddrEnc, val.JSONAddrEnc, + val.GeomAddrEnc, val.CommitAddrEnc, + val.BytesAdaptiveEnc, val.GeomAdaptiveEnc: + return placeholderBinary + } + + return td.FormatValue(ctx, i, td.GetField(i, tup)) +}