diff --git a/CHANGELOG.md b/CHANGELOG.md index e6bd33e48..e863661cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Now gitbase uses [go-borges](https://github.com/src-d/go-borges) to access repositories + - The type of files in each directory has to be specified ([#867](https://github.com/src-d/gitbase/pull/867)) + - Supports new rooted repository format and separates references and objects from each repo (https://github.com/src-d/borges/issues/389) +- Changed cli to be able to specify different formats ([#866](https://github.com/src-d/gitbase/issues/866)) + ## [0.21.0-beta3] - 2019-06-19 ### Fixed diff --git a/_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva b/_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva index 9af2d1251..03d997a60 100644 Binary files a/_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva and b/_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva differ diff --git a/_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva b/_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva index 135b0be94..c9d0ab116 100644 Binary files a/_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva and b/_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva differ diff --git a/_testdata/ff/fff840f8784ef162dc83a1465fc5763d890b68ba.siva b/_testdata/fff840f8784ef162dc83a1465fc5763d890b68ba.siva similarity index 99% rename from _testdata/ff/fff840f8784ef162dc83a1465fc5763d890b68ba.siva rename to _testdata/fff840f8784ef162dc83a1465fc5763d890b68ba.siva index 6a9f28969..da4260fdd 100644 Binary files a/_testdata/ff/fff840f8784ef162dc83a1465fc5763d890b68ba.siva and b/_testdata/fff840f8784ef162dc83a1465fc5763d890b68ba.siva differ diff --git a/blobs.go b/blobs.go index 3a2a23128..5d3098bc0 100644 --- a/blobs.go +++ b/blobs.go @@ -209,7 +209,7 @@ func (i *blobRowIter) nextByHash() (sql.Row, error) { return nil, err } - return blobToRow(i.repo.ID, blob, i.readContent) + return blobToRow(i.repo.ID(), blob, i.readContent) } } @@ -236,7 +236,7 @@ func (i *blobRowIter) next() (sql.Row, error) { return nil, err } - return blobToRow(i.repo.ID, o, i.readContent) + return blobToRow(i.repo.ID(), o, i.readContent) } } @@ -347,8 +347,7 @@ func newBlobsKeyValueIter( return nil, err } - r := pool.repositories[repo.ID] - idx, err := newRepositoryIndex(r) + idx, err := newRepositoryIndex(repo) if err != nil { return nil, err } @@ -379,7 +378,7 @@ func (i *blobsKeyValueIter) Next() ([]interface{}, []byte, error) { } key, err := encodeIndexKey(&packOffsetIndexKey{ - Repository: i.repo.ID, + Repository: i.repo.ID(), Packfile: packfile.String(), Offset: offset, Hash: hash, @@ -388,7 +387,7 @@ func (i *blobsKeyValueIter) Next() ([]interface{}, []byte, error) { return nil, nil, err } - row, err := blobToRow(i.repo.ID, blob, stringContains(i.columns, "blob_content")) + row, err := blobToRow(i.repo.ID(), blob, stringContains(i.columns, "blob_content")) if err != nil { return nil, nil, err } diff --git a/blobs_test.go b/blobs_test.go index ee5c86269..39c5a3f81 100644 --- a/blobs_test.go +++ b/blobs_test.go @@ -3,9 +3,9 @@ package gitbase import ( "testing" - "github.com/stretchr/testify/require" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" ) func TestBlobsTable(t *testing.T) { diff --git a/checksum.go b/checksum.go index bcb1b7fd8..fd3cd5a8f 100644 --- a/checksum.go +++ b/checksum.go @@ -8,7 +8,6 @@ import ( "sort" "strings" - git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -18,9 +17,23 @@ type checksumable struct { func (c *checksumable) Checksum() (string, error) { hash := sha1.New() - for _, id := range c.pool.idOrder { - repo := c.pool.repositories[id] - hash.Write([]byte(id)) + iter, err := c.pool.RepoIter() + if err != nil { + return "", err + } + defer iter.Close() + + var checksums checksums + for { + hash.Reset() + + repo, err := iter.Next() + if err == io.EOF { + break + } + if err != nil { + return "", err + } bytes, err := readChecksum(repo) if err != nil { @@ -39,12 +52,28 @@ func (c *checksumable) Checksum() (string, error) { if _, err = hash.Write(bytes); err != nil { return "", err } + + c := checksum{ + name: repo.ID(), + hash: hash.Sum(nil), + } + + checksums = append(checksums, c) + } + + sort.Stable(checksums) + hash.Reset() + + for _, c := range checksums { + if _, err = hash.Write(c.hash); err != nil { + return "", err + } } return base64.StdEncoding.EncodeToString(hash.Sum(nil)), nil } -func readChecksum(r repository) ([]byte, error) { +func readChecksum(r *Repository) ([]byte, error) { fs, err := r.FS() if err != nil { return nil, err @@ -99,15 +128,23 @@ func (b byHashAndName) Less(i, j int) bool { return strings.Compare(b[i].name, b[j].name) < 0 } -func readRefs(r repository) ([]byte, error) { - repo, err := r.Repo() - if err != nil { - if err == git.ErrRepositoryNotExists { - return nil, nil - } - return nil, err +type checksum struct { + name string + hash []byte +} + +type checksums []checksum + +func (b checksums) Len() int { return len(b) } +func (b checksums) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b checksums) Less(i, j int) bool { + if cmp := bytes.Compare(b[i].hash, b[j].hash); cmp != 0 { + return cmp < 0 } + return strings.Compare(b[i].name, b[j].name) < 0 +} +func readRefs(repo *Repository) ([]byte, error) { buf := bytes.NewBuffer(nil) refs, err := repo.References() diff --git a/checksum_test.go b/checksum_test.go index 8a3044a2a..5635b1179 100644 --- a/checksum_test.go +++ b/checksum_test.go @@ -8,7 +8,13 @@ import ( fixtures "github.com/src-d/go-git-fixtures" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-billy.v4/osfs" +) + +const ( + checksumMulti = "W/lxpR0jZ6O6BqVANTYTDMlAS/4=" + checksumSingle = "zqLF31JlrtJ57XNC+cQ+2hSkBkw=" + checksumSiva = "X27U+Lww5UOk1+/21bVFgI4uJyM=" ) func TestChecksum(t *testing.T) { @@ -18,32 +24,40 @@ func TestChecksum(t *testing.T) { require.NoError(fixtures.Clean()) }() - pool := NewRepositoryPool(cache.DefaultMaxSize) + lib, pool, err := newMultiPool() + require.NoError(err) for i, f := range fixtures.ByTag("worktree") { path := f.Worktree().Root() - require.NoError(pool.AddGitWithID(fmt.Sprintf("repo_%d", i), path)) + require.NoError(lib.AddPlain(fmt.Sprintf("repo_%d", i), path, nil)) } c := &checksumable{pool} checksum, err := c.Checksum() require.NoError(err) - require.Equal("mGPoKCyOIkXX4reGe1vTBPIOg2E=", checksum) + require.Equal(checksumMulti, checksum) - pool = NewRepositoryPool(cache.DefaultMaxSize) + lib, pool, err = newMultiPool() + require.NoError(err) path := fixtures.ByTag("worktree").One().Worktree().Root() - require.NoError(pool.AddGitWithID("worktree", path)) + require.NoError(lib.AddPlain("worktree", path, nil)) c = &checksumable{pool} checksum, err = c.Checksum() require.NoError(err) - require.Equal("rwQnBj7HRazv9wuU//nQ+nuf0WY=", checksum) + require.Equal(checksumSingle, checksum) } func TestChecksumSiva(t *testing.T) { require := require.New(t) - pool := NewRepositoryPool(cache.DefaultMaxSize) + lib, pool, err := newMultiPool() + require.NoError(err) + + cwd, err := os.Getwd() + require.NoError(err) + cwdFS := osfs.New(cwd) + require.NoError( filepath.Walk("_testdata", func(path string, info os.FileInfo, err error) error { if err != nil { @@ -51,7 +65,7 @@ func TestChecksumSiva(t *testing.T) { } if IsSivaFile(path) { - require.NoError(pool.AddSivaFile(path)) + require.NoError(lib.AddSiva(path, cwdFS)) } return nil @@ -61,7 +75,7 @@ func TestChecksumSiva(t *testing.T) { c := &checksumable{pool} checksum, err := c.Checksum() require.NoError(err) - require.Equal("wJEvZNAc7QRszsf9KhGu+UeKto0=", checksum) + require.Equal(checksumSiva, checksum) } func TestChecksumStable(t *testing.T) { @@ -71,11 +85,12 @@ func TestChecksumStable(t *testing.T) { require.NoError(fixtures.Clean()) }() - pool := NewRepositoryPool(cache.DefaultMaxSize) + lib, pool, err := newMultiPool() + require.NoError(err) for i, f := range fixtures.ByTag("worktree") { path := f.Worktree().Root() - require.NoError(pool.AddGitWithID(fmt.Sprintf("repo_%d", i), path)) + require.NoError(lib.AddPlain(fmt.Sprintf("repo_%d", i), path, nil)) } c := &checksumable{pool} @@ -83,6 +98,6 @@ func TestChecksumStable(t *testing.T) { for i := 0; i < 100; i++ { checksum, err := c.Checksum() require.NoError(err) - require.Equal("mGPoKCyOIkXX4reGe1vTBPIOg2E=", checksum) + require.Equal(checksumMulti, checksum) } } diff --git a/cmd/gitbase/command/server.go b/cmd/gitbase/command/server.go index accc26600..e30ef6543 100644 --- a/cmd/gitbase/command/server.go +++ b/cmd/gitbase/command/server.go @@ -3,8 +3,10 @@ package command import ( "fmt" "net" + "net/url" "os" "path/filepath" + "regexp" "runtime" "strconv" "strings" @@ -16,6 +18,10 @@ import ( "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" + "github.com/src-d/go-borges" + "github.com/src-d/go-borges/libraries" + "github.com/src-d/go-borges/plain" + "github.com/src-d/go-borges/siva" sqle "github.com/src-d/go-mysql-server" "github.com/src-d/go-mysql-server/auth" "github.com/src-d/go-mysql-server/server" @@ -23,6 +29,7 @@ import ( "github.com/src-d/go-mysql-server/sql/analyzer" "github.com/src-d/go-mysql-server/sql/index/pilosa" "github.com/uber/jaeger-client-go/config" + "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-git.v4/plumbing/cache" "vitess.io/vitess/go/mysql" ) @@ -42,10 +49,17 @@ type Server struct { pool *gitbase.RepositoryPool userAuth auth.Auth + rootLibrary *libraries.Libraries + plainLibrary *plain.Library + sharedCache cache.Object + Name string `long:"db" default:"gitbase" description:"Database name"` Version string // Version of the application. - Directories []string `short:"d" long:"directories" description:"Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs."` - Depth int `long:"depth" default:"1000" description:"load repositories looking at less than nested subdirectories."` + Directories []string `short:"d" long:"directories" description:"Path where standard git repositories are located, multiple directories can be defined."` + Format string `long:"format" default:"git" choice:"git" choice:"siva" description:"Library format"` + Bucket int `long:"bucket" default:"2" description:"Bucketing level to use with siva libraries"` + Bare bool `long:"bare" description:"Sets the library to use bare git repositories, used only with git format libraries"` + NonRooted bool `long:"non-rooted" description:"Disables treating siva files as rooted repositories"` Host string `long:"host" default:"localhost" description:"Host where the server is going to listen"` Port int `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"` User string `short:"u" long:"user" default:"root" description:"User name used for connection"` @@ -59,8 +73,6 @@ type Server struct { TraceEnabled bool `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"` ReadOnly bool `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well. Cannot be used with --user-file." env:"GITBASE_READONLY"` SkipGitErrors bool // SkipGitErrors disables failing when Git errors are found. - DisableGit bool `long:"no-git" description:"disable the load of git standard repositories."` - DisableSiva bool `long:"no-siva" description:"disable the load of siva files."` Verbose bool `short:"v" description:"Activates the verbose mode"` LogLevel string `long:"log-level" env:"GITBASE_LOG_LEVEL" choice:"info" choice:"debug" choice:"warning" choice:"error" choice:"fatal" default:"info" description:"logging level"` } @@ -208,7 +220,10 @@ func (c *Server) buildDatabase() error { ) } - c.pool = gitbase.NewRepositoryPool(c.CacheSize * cache.MiByte) + c.rootLibrary = libraries.New(libraries.Options{}) + c.pool = gitbase.NewRepositoryPool(c.CacheSize*cache.MiByte, c.rootLibrary) + + c.sharedCache = cache.NewObjectLRU(512 * cache.MiByte) if err := c.addDirectories(); err != nil { return err @@ -252,25 +267,25 @@ func (c *Server) registerDrivers() error { func (c *Server) addDirectories() error { if len(c.Directories) == 0 { - logrus.Error("At least one folder should be provided.") - } - - if c.DisableGit && c.DisableSiva { - logrus.Warn("The load of git repositories and siva files are disabled," + - " no repository will be added.") - - return nil + logrus.Error("at least one folder should be provided.") } - if c.Depth < 1 { - logrus.Warn("--depth flag set to a number less than 1," + - " no repository will be added.") + for _, d := range c.Directories { + dir := directory{ + Path: d, + Format: c.Format, + Bare: c.Bare, + Bucket: c.Bucket, + Rooted: !c.NonRooted, + } - return nil - } + dir, err := parseDirectory(dir) + if err != nil { + return err + } - for _, directory := range c.Directories { - if err := c.addDirectory(directory); err != nil { + err = c.addDirectory(dir) + if err != nil { return err } } @@ -278,113 +293,136 @@ func (c *Server) addDirectories() error { return nil } -func (c *Server) addDirectory(directory string) error { - matches, err := gitbase.PatternMatches(directory) - if err != nil { - return err - } +func (c *Server) addDirectory(d directory) error { + if d.Format == "siva" { + sivaOpts := siva.LibraryOptions{ + Transactional: true, + RootedRepo: d.Rooted, + Cache: c.sharedCache, + Bucket: d.Bucket, + Performance: true, + RegistryCache: 100000, + } - for _, match := range matches { - if err := c.addMatch(directory, match); err != nil { - logrus.WithFields(logrus.Fields{ - "path": match, - "error": err, - }).Error("path couldn't be inspected") + lib, err := siva.NewLibrary(d.Path, osfs.New(d.Path), sivaOpts) + if err != nil { + return err } - } - return nil -} + err = c.rootLibrary.Add(lib) + if err != nil { + return err + } -func (c *Server) addMatch(prefix, match string) error { - root, err := filepath.Abs(match) - if err != nil { - return err + return nil } - root, err = filepath.EvalSymlinks(root) - if err != nil { - return err + plainOpts := &plain.LocationOptions{ + Cache: c.sharedCache, + Performance: true, + Bare: d.Bare, } - initDepth := strings.Count(root, string(os.PathSeparator)) - return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if c.plainLibrary == nil { + c.plainLibrary = plain.NewLibrary(borges.LibraryID("plain")) + err := c.rootLibrary.Add(c.plainLibrary) if err != nil { - if os.IsPermission(err) { - return filepath.SkipDir - } return err } + } - if info.IsDir() { - if err := c.addIfGitRepo(prefix, path); err != nil { - return err - } - - depth := strings.Count(path, string(os.PathSeparator)) - initDepth - if depth >= c.Depth { - return filepath.SkipDir - } + loc, err := plain.NewLocation( + borges.LocationID(d.Path), + osfs.New(d.Path), + plainOpts) + if err != nil { + return err + } - return nil - } + c.plainLibrary.AddLocation(loc) - if !c.DisableSiva && - info.Mode().IsRegular() && - gitbase.IsSivaFile(info.Name()) { - id, err := gitbase.StripPrefix(prefix, path) - if err != nil { - return err - } + return nil +} - if err := c.pool.AddSivaFileWithID(id, path); err != nil { - logrus.WithFields(logrus.Fields{ - "path": path, - "error": err, - }).Error("repository could not be addded") +type directory struct { + Path string + Format string + Bucket int + Rooted bool + Bare bool +} - return nil - } +var ( + uriReg = regexp.MustCompile(`^\w+:.*`) + ErrInvalid = fmt.Errorf("invalid option") +) - logrus.WithField("path", path).Debug("repository added") - } +func parseDirectory(dir directory) (directory, error) { + d := dir.Path - return nil - }) -} + if !uriReg.Match([]byte(d)) { + return dir, nil + } -func (c *Server) addIfGitRepo(prefix, path string) error { - ok, err := gitbase.IsGitRepo(path) + u, err := url.ParseRequestURI(d) if err != nil { - logrus.WithFields(logrus.Fields{ - "path": path, - "error": err, - }).Error("path couldn't be inspected") + logrus.Errorf("invalid directory format %v", d) + return dir, err + } - return filepath.SkipDir + if u.Scheme != "file" { + logrus.Errorf("only file scheme is supported: %v", d) + return dir, fmt.Errorf("scheme not suported in directory %v", d) } - if ok { - if !c.DisableGit { - id, err := gitbase.StripPrefix(prefix, path) - if err != nil { - return err + dir.Path = filepath.Join(u.Hostname(), u.Path) + query := u.Query() + + for k, v := range query { + if len(v) != 1 { + logrus.Errorf("invalid number of options for %v", v) + return dir, ErrInvalid + } + + val := v[0] + switch strings.ToLower(k) { + case "format": + if val != "siva" && val != "git" { + logrus.Errorf("invalid value in format, it can only "+ + "be siva or git %v", val) + return dir, ErrInvalid } + dir.Format = val - if err := c.pool.AddGitWithID(id, path); err != nil { - logrus.WithFields(logrus.Fields{ - "id": id, - "path": path, - "error": err, - }).Error("repository could not be added") + case "bare": + if val != "true" && val != "false" { + logrus.Errorf("invalid value in bare, it can only "+ + "be true or false %v", val) + return dir, ErrInvalid } + dir.Bare = (val == "true") - logrus.WithField("path", path).Debug("repository added") - } + case "rooted": + if val != "true" && val != "false" { + logrus.Errorf("invalid value in rooted, it can only "+ + "be true or false %v", val) + return dir, ErrInvalid + } + dir.Rooted = (val == "true") - // either the repository is added or not, the path must be skipped - return filepath.SkipDir + case "bucket": + num, err := strconv.Atoi(val) + if err != nil { + logrus.Errorf("invalid value in bucket: %v", val) + return dir, ErrInvalid + } + dir.Bucket = num + + default: + logrus.Errorf("invalid option: %v", k) + return dir, ErrInvalid + } } - return nil + return dir, nil } diff --git a/cmd/gitbase/command/server_test.go b/cmd/gitbase/command/server_test.go index 241437ed9..5adf3a375 100644 --- a/cmd/gitbase/command/server_test.go +++ b/cmd/gitbase/command/server_test.go @@ -1,65 +1,149 @@ package command import ( - "io" - "os" - "sort" "testing" - "github.com/src-d/gitbase" "github.com/stretchr/testify/require" ) -func TestAddMatch(t *testing.T) { - require := require.New(t) - - notPermissionDir := "../../../_testdata/not-permission/" - fi, err := os.Stat(notPermissionDir) - require.NoError(err) - - require.NoError(os.Chmod(notPermissionDir, 0)) - defer func() { - require.NoError(os.Chmod(notPermissionDir, fi.Mode())) - }() - - expected := []struct { - path string - err func(error, ...interface{}) +func TestDirectories(t *testing.T) { + tests := []struct { + path string + expected directory + error bool }{ - {"../../../_testdata/repositories/", require.NoError}, - {"../../../_testdata/repositories-link/", require.NoError}, - {notPermissionDir, require.NoError}, - {"../../../_testdata/repositories-not-exist/", require.Error}, - } - c := &Server{pool: gitbase.NewRepositoryPool(0)} - for _, e := range expected { - e.err(c.addMatch("../../../_testdata", e.path)) + { + path: "relative", + expected: directory{Path: "relative"}, + }, + { + path: "longer/relative", + expected: directory{Path: "longer/relative"}, + }, + { + path: "/absolute", + expected: directory{Path: "/absolute"}, + }, + { + path: "/longer/absolute", + expected: directory{Path: "/longer/absolute"}, + }, + { + path: "file://relative", + expected: directory{Path: "relative"}, + }, + { + path: "file://longer/relative", + expected: directory{Path: "longer/relative"}, + }, + { + path: "file:///absolute", + expected: directory{Path: "/absolute"}, + }, + { + path: "file:///longer/absolute", + expected: directory{Path: "/longer/absolute"}, + }, + { + path: "http://relative", + error: true, + }, + { + path: "file:///siva/path?format=siva", + expected: directory{ + Path: "/siva/path", + Format: "siva", + }, + }, + { + path: "file:///siva/path?format=git", + expected: directory{ + Path: "/siva/path", + Format: "git", + }, + }, + { + path: "file:///siva/path?format=nope", + error: true, + }, + { + path: "file:///siva/path?bare=true", + expected: directory{ + Path: "/siva/path", + Bare: true, + }, + }, + { + path: "file:///siva/path?bare=false", + expected: directory{ + Path: "/siva/path", + Bare: false, + }, + }, + { + path: "file:///siva/path?bare=nope", + error: true, + }, + { + path: "file:///siva/path?rooted=true", + expected: directory{ + Path: "/siva/path", + Rooted: true, + }, + }, + { + path: "file:///siva/path?rooted=false", + expected: directory{ + Path: "/siva/path", + Rooted: false, + }, + }, + { + path: "file:///siva/path?bare=nope", + error: true, + }, + { + path: "file:///siva/path?bucket=42", + expected: directory{ + Path: "/siva/path", + Bucket: 42, + }, + }, + { + path: "file:///siva/path?bucket=false", + error: true, + }, + { + path: "file:///siva/path?format=git&bare=false", + expected: directory{ + Path: "/siva/path", + Format: "git", + Bare: false, + }, + }, + { + path: "file:///siva/path?format=siva&rooted=false&bucket=42", + expected: directory{ + Path: "/siva/path", + Format: "siva", + Rooted: false, + Bucket: 42, + }, + }, } -} - -func TestAddDirectory(t *testing.T) { - require := require.New(t) - c := &Server{pool: gitbase.NewRepositoryPool(0), Depth: 5} - require.NoError(c.addDirectory("../../../_testdata/*")) - i, err := c.pool.RepoIter() - require.NoError(err) - var repositories []string - for { - r, err := i.Next() - if err == io.EOF { - require.NoError(i.Close()) - break - } + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + require := require.New(t) + dir := directory{Path: test.path} - repositories = append(repositories, r.ID) - } + dir, err := parseDirectory(dir) + if test.error { + require.Error(err) + return + } - sort.Strings(repositories) - expected := []string{ - "05893125684f2d3943cd84a7ab2b75e53668fba1.siva", - "ff/fff840f8784ef162dc83a1465fc5763d890b68ba.siva", - "fff7062de8474d10a67d417ccea87ba6f58ca81d.siva", + require.Equal(test.expected, dir) + }) } - require.Equal(expected, repositories) } diff --git a/commit_blobs.go b/commit_blobs.go index 150020247..0b9419adc 100644 --- a/commit_blobs.go +++ b/commit_blobs.go @@ -84,7 +84,7 @@ func (t *commitBlobsTable) PartitionRows( return nil, err } - if len(repos) > 0 && !stringContains(repos, repo.ID) { + if len(repos) > 0 && !stringContains(repos, repo.ID()) { return noRows, nil } @@ -307,7 +307,7 @@ func (i *commitBlobsRowIter) next() (sql.Row, error) { } return sql.NewRow( - i.repo.ID, i.currCommit.Hash.String(), file.Blob.Hash.String(), + i.repo.ID(), i.currCommit.Hash.String(), file.Blob.Hash.String(), ), nil } } diff --git a/commit_blobs_test.go b/commit_blobs_test.go index 0902d9beb..17f3c95f1 100644 --- a/commit_blobs_test.go +++ b/commit_blobs_test.go @@ -4,11 +4,11 @@ import ( "io" "testing" - "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" "github.com/src-d/go-mysql-server/sql/plan" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing" ) func TestCommitBlobsTableRowIter(t *testing.T) { diff --git a/commit_files.go b/commit_files.go index f5163bddd..f25a02dab 100644 --- a/commit_files.go +++ b/commit_files.go @@ -90,7 +90,7 @@ func (t *commitFilesTable) PartitionRows( return nil, err } - if len(repos) > 0 && !stringContains(repos, repo.ID) { + if len(repos) > 0 && !stringContains(repos, repo.ID()) { return noRows, nil } @@ -257,7 +257,7 @@ func (i *commitFilesRowIter) next() (sql.Row, error) { if err != nil { if i.skipGitErrors && err != io.EOF { logrus.WithFields(logrus.Fields{ - "repo": i.repo.ID, + "repo": i.repo.ID(), "err": err, }).Error("skipped commit in commit_files") continue @@ -269,7 +269,7 @@ func (i *commitFilesRowIter) next() (sql.Row, error) { if err != nil { if i.skipGitErrors { logrus.WithFields(logrus.Fields{ - "repo": i.repo.ID, + "repo": i.repo.ID(), "err": err, "commit": i.commit.Hash.String(), }).Error("can't get files for commit") @@ -287,7 +287,7 @@ func (i *commitFilesRowIter) next() (sql.Row, error) { if i.skipGitErrors { logrus.WithFields(logrus.Fields{ - "repo": i.repo.ID, + "repo": i.repo.ID(), "err": err, "commit": i.commit.Hash.String(), }).Error("can't get next file for commit") @@ -307,7 +307,7 @@ func (i *commitFilesRowIter) next() (sql.Row, error) { func newCommitFilesRow(repo *Repository, commit *object.Commit, file *object.File) sql.Row { return sql.NewRow( - repo.ID, + repo.ID(), commit.Hash.String(), file.Name, file.Blob.Hash.String(), @@ -435,8 +435,7 @@ func newCommitFilesKeyValueIter( repo *Repository, columns []string, ) (sql.IndexKeyValueIter, error) { - r := pool.repositories[repo.ID] - idx, err := newRepositoryIndex(r) + idx, err := newRepositoryIndex(repo) if err != nil { return nil, err } @@ -486,7 +485,7 @@ func (i *commitFilesKeyValueIter) Next() ([]interface{}, []byte, error) { } key, err := encodeIndexKey(&commitFileIndexKey{ - Repository: i.repo.ID, + Repository: i.repo.ID(), Packfile: packfile.String(), Hash: f.Blob.Hash.String(), Offset: offset, diff --git a/commit_files_test.go b/commit_files_test.go index 3272e38d5..9c879ddda 100644 --- a/commit_files_test.go +++ b/commit_files_test.go @@ -4,11 +4,11 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" - "github.com/src-d/go-mysql-server/sql" - "github.com/src-d/go-mysql-server/sql/expression" ) func TestCommitFilesTableRowIter(t *testing.T) { diff --git a/commit_trees.go b/commit_trees.go index 33e43d118..543d7824c 100644 --- a/commit_trees.go +++ b/commit_trees.go @@ -85,7 +85,7 @@ func (t *commitTreesTable) PartitionRows( return nil, err } - if len(repos) > 0 && !stringContains(repos, repo.ID) { + if len(repos) > 0 && !stringContains(repos, repo.ID()) { return noRows, nil } @@ -293,7 +293,7 @@ func (i *commitTreesRowIter) next() (sql.Row, error) { if tree != nil { return sql.NewRow( - i.repo.ID, + i.repo.ID(), i.commit.Hash.String(), tree.Hash.String(), ), nil @@ -316,7 +316,7 @@ func (i *commitTreesRowIter) next() (sql.Row, error) { } return sql.NewRow( - i.repo.ID, + i.repo.ID(), i.commit.Hash.String(), entry.Hash.String(), ), nil diff --git a/commit_trees_test.go b/commit_trees_test.go index b25b03293..26832eee0 100644 --- a/commit_trees_test.go +++ b/commit_trees_test.go @@ -3,10 +3,10 @@ package gitbase import ( "testing" - "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing" ) func TestCommitTreesRowIter(t *testing.T) { diff --git a/commits.go b/commits.go index f89650a0a..4f4b0bc2b 100644 --- a/commits.go +++ b/commits.go @@ -180,7 +180,7 @@ func (i *commitRowIter) Next() (sql.Row, error) { return nil, err } - return commitToRow(i.repo.ID, c), nil + return commitToRow(i.repo.ID(), c), nil } } @@ -409,8 +409,7 @@ func newCommitsKeyValueIter( columns []string, ) (sql.IndexKeyValueIter, error) { var err error - r := pool.repositories[repo.ID] - idx, err := newRepositoryIndex(r) + idx, err := newRepositoryIndex(repo) if err != nil { return nil, err } @@ -445,7 +444,7 @@ func (i *commitsKeyValueIter) Next() ([]interface{}, []byte, error) { } key, err := encodeIndexKey(&packOffsetIndexKey{ - Repository: i.repo.ID, + Repository: i.repo.ID(), Packfile: packfile.String(), Offset: offset, Hash: hash, @@ -454,7 +453,7 @@ func (i *commitsKeyValueIter) Next() ([]interface{}, []byte, error) { return nil, nil, err } - row := commitToRow(i.repo.ID, commit) + row := commitToRow(i.repo.ID(), commit) values, err := rowIndexValues(row, i.columns, CommitsSchema) if err != nil { return nil, nil, err diff --git a/commits_test.go b/commits_test.go index f92e45b28..06fa93649 100644 --- a/commits_test.go +++ b/commits_test.go @@ -3,9 +3,9 @@ package gitbase import ( "testing" - "github.com/stretchr/testify/require" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" ) func TestCommitsTable(t *testing.T) { diff --git a/common_test.go b/common_test.go index 6e514b4f4..e2ba9d737 100644 --- a/common_test.go +++ b/common_test.go @@ -3,23 +3,24 @@ package gitbase import ( "context" "io" - "io/ioutil" "os" "path/filepath" "strings" "sync" "testing" + "github.com/src-d/go-borges" + "github.com/src-d/go-borges/libraries" + "github.com/src-d/go-borges/plain" + "github.com/src-d/go-borges/siva" fixtures "github.com/src-d/go-git-fixtures" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - sivafs "gopkg.in/src-d/go-billy-siva.v4" - billy "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-billy.v4/osfs" - git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing/cache" - "gopkg.in/src-d/go-git.v4/storage/filesystem" ) type CleanupFunc func() @@ -38,21 +39,27 @@ func setupRepos(t *testing.T) (*sql.Context, []string, CleanupFunc) { return buildSession(t, fixtures.ByTag("worktree")) } -func buildSession(t *testing.T, repos fixtures.Fixtures, +func buildSession( + t *testing.T, + repos fixtures.Fixtures, ) (ctx *sql.Context, paths []string, cleanup CleanupFunc) { require := require.New(t) t.Helper() - pool := NewRepositoryPool(cache.DefaultMaxSize) + lib, err := newMultiLibrary() + require.NoError(err) + + pool := NewRepositoryPool(cache.DefaultMaxSize, lib) for _, fixture := range repos { path := fixture.Worktree().Root() ok, err := IsGitRepo(path) require.NoError(err) if ok { - if err := pool.AddGit(path); err == nil { - _, err := pool.GetRepo(path) + name := strings.TrimLeft(path, string(os.PathSeparator)) + if err := lib.AddPlain(name, path, nil); err == nil { + _, err := pool.GetRepo(name) require.NoError(err) - paths = append(paths, path) + paths = append(paths, pathToName(path)) } } } @@ -78,131 +85,108 @@ The following code adds utilities to test that siva files are properly closed. Instead of using normal setup you can use setupSivaCloseRepos, it returns a context with a pool with all the sivas in "_testdata" directory. It also tracks all siva filesystems opened. Its closed state can be checked with -closedSiva.Check(). +closedLib.Check(). */ -type closedSiva struct { - closed []bool - m sync.Mutex +type closedRepository struct { + borges.Repository + closed bool } -func (c *closedSiva) NewFS(path string) (billy.Filesystem, error) { - c.m.Lock() - defer c.m.Unlock() - - localfs := osfs.New(filepath.Dir(path)) - - tmpDir, err := ioutil.TempDir(os.TempDir(), "gitbase-siva") - if err != nil { - return nil, err - } - - tmpfs := osfs.New(tmpDir) +func (c *closedRepository) Close() error { + c.closed = true + return c.Repository.Close() +} - fs, err := sivafs.NewFilesystem(localfs, filepath.Base(path), tmpfs) - if err != nil { - return nil, err - } +type closedLibrary struct { + *multiLibrary + repos []*closedRepository + m sync.Mutex +} - pos := len(c.closed) - c.closed = append(c.closed, false) +func (c *closedLibrary) trackRepo(r borges.Repository) *closedRepository { + c.m.Lock() + defer c.m.Unlock() - fun := func() { - c.m.Lock() - defer c.m.Unlock() - c.closed[pos] = true - } + closed := &closedRepository{Repository: r} + c.repos = append(c.repos, closed) - return &closedSivaFilesystem{fs, fun}, nil + return closed } -func (c *closedSiva) Check() bool { - for _, f := range c.closed { - if !f { +func (c *closedLibrary) Check() bool { + for _, r := range c.repos { + if !r.closed { return false } } - return true } -type closedSivaFilesystem struct { - sivafs.SivaFS - closeFunc func() -} - -func (c *closedSivaFilesystem) Sync() error { - if c.closeFunc != nil { - c.closeFunc() - } - - return c.SivaFS.Sync() -} - -var _ repository = new(closedSivaRepository) - -type closedSivaRepository struct { - path string - siva *closedSiva - cache cache.Object -} - -func (c *closedSivaRepository) ID() string { - return c.path -} - -func (c *closedSivaRepository) Repo() (*Repository, error) { - fs, err := c.FS() +func (c *closedLibrary) Get( + r borges.RepositoryID, + m borges.Mode, +) (borges.Repository, error) { + repo, err := c.multiLibrary.Get(r, m) if err != nil { return nil, err } - s := fs.(*closedSivaFilesystem) - closeFunc := func() { s.Sync() } + return c.trackRepo(repo), nil +} - sto := filesystem.NewStorageWithOptions(fs, c.Cache(), gitStorerOptions) - repo, err := git.Open(sto, nil) +func (c *closedLibrary) Repositories(m borges.Mode) (borges.RepositoryIterator, error) { + iter, err := c.multiLibrary.Repositories(m) if err != nil { return nil, err - } - return NewRepository(c.path, repo, closeFunc), nil -} - -func (c *closedSivaRepository) FS() (billy.Filesystem, error) { - return c.siva.NewFS(c.path) + return &closedIter{ + RepositoryIterator: iter, + c: c, + }, nil } -func (c *closedSivaRepository) Path() string { - return c.path +type closedIter struct { + borges.RepositoryIterator + c *closedLibrary } -func (c *closedSivaRepository) Cache() cache.Object { - if c.cache == nil { - c.cache = cache.NewObjectLRUDefault() +func (i *closedIter) Next() (borges.Repository, error) { + repo, err := i.RepositoryIterator.Next() + if err != nil { + return nil, err } - return c.cache + return i.c.trackRepo(repo), nil } // setupSivaCloseRepos creates a pool with siva files that can be checked // if they've been closed. -func setupSivaCloseRepos(t *testing.T, dir string) (*sql.Context, *closedSiva) { +func setupSivaCloseRepos(t *testing.T, dir string) (*sql.Context, *closedLibrary) { require := require.New(t) t.Helper() - cs := new(closedSiva) - pool := NewRepositoryPool(cache.DefaultMaxSize) + lib, err := newMultiLibrary() + require.NoError(err) + + closedLib := &closedLibrary{multiLibrary: lib} + pool := NewRepositoryPool(cache.DefaultMaxSize, closedLib) + + cwd, err := os.Getwd() + require.NoError(err) + cwdFS := osfs.New(cwd) filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if strings.HasSuffix(path, ".siva") { - repo := &closedSivaRepository{path: path, siva: cs} - err := pool.Add(repo) - require.NoError(err) + if err != nil { + return err + } + + if IsSivaFile(path) { + require.NoError(lib.AddSiva(path, cwdFS)) } return nil @@ -212,7 +196,7 @@ func setupSivaCloseRepos(t *testing.T, dir string) (*sql.Context, *closedSiva) { session := NewSession(pool, WithSkipGitErrors(true)) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) - return ctx, cs + return ctx, closedLib } func testTableIndexIterClosed(t *testing.T, table sql.IndexableTable) { @@ -292,3 +276,206 @@ func poolFromCtx(t *testing.T, ctx *sql.Context) *RepositoryPool { return session.Pool } + +func pathToName(path string) string { + return strings.TrimLeft(path, string(os.PathSeparator)) +} + +type multiLibrary struct { + lib *libraries.Libraries + + plain *plain.Library + plainFS billy.Filesystem + siva *siva.Library + sivaFS billy.Filesystem + + rootFS billy.Filesystem +} + +func newMultiPool() (*multiLibrary, *RepositoryPool, error) { + lib, err := newMultiLibrary() + if err != nil { + return nil, nil, err + } + + pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + + return lib, pool, err +} + +func newMultiLibrary() (*multiLibrary, error) { + plainFS := memfs.New() + plainLoc, err := plain.NewLocation("root", plainFS, nil) + if err != nil { + return nil, err + } + plainLib := plain.NewLibrary("plain") + plainLib.AddLocation(plainLoc) + + sivaFS := memfs.New() + sivaLib, err := siva.NewLibrary("siva", sivaFS, siva.LibraryOptions{ + RootedRepo: true, + }) + if err != nil { + return nil, err + } + + libs := libraries.New(libraries.Options{}) + + err = libs.Add(plainLib) + if err != nil { + return nil, err + } + err = libs.Add(sivaLib) + if err != nil { + return nil, err + } + + return &multiLibrary{ + lib: libs, + plain: plainLib, + plainFS: plainFS, + siva: sivaLib, + sivaFS: sivaFS, + rootFS: osfs.New("/"), + }, nil +} + +func (m *multiLibrary) AddPlain( + name string, + path string, + fs billy.Filesystem, +) error { + if fs == nil { + fs = m.rootFS + } + + return recursiveCopy(name, m.plainFS, path, fs) +} + +func (m *multiLibrary) AddSiva( + path string, + fs billy.Filesystem, +) error { + if fs == nil { + fs = m.rootFS + } + + name := filepath.Base(path) + return recursiveCopy(name, m.sivaFS, path, fs) +} + +func (m *multiLibrary) ID() borges.LibraryID { + return m.lib.ID() +} + +func (m *multiLibrary) Init(r borges.RepositoryID) (borges.Repository, error) { + return m.lib.Init(r) +} + +func (m *multiLibrary) Get(r borges.RepositoryID, mode borges.Mode) (borges.Repository, error) { + return m.lib.Get(r, mode) +} + +func (m *multiLibrary) GetOrInit(r borges.RepositoryID) (borges.Repository, error) { + return m.lib.GetOrInit(r) +} + +func (m *multiLibrary) Has(r borges.RepositoryID) (bool, borges.LibraryID, borges.LocationID, error) { + return m.lib.Has(r) +} + +func (m *multiLibrary) Repositories(mode borges.Mode) (borges.RepositoryIterator, error) { + return m.lib.Repositories(mode) +} + +func (m *multiLibrary) Location(r borges.LocationID) (borges.Location, error) { + return m.lib.Location(r) +} + +func (m *multiLibrary) Locations() (borges.LocationIterator, error) { + return m.lib.Locations() +} + +func (m *multiLibrary) Library(l borges.LibraryID) (borges.Library, error) { + return m.lib.Library(l) +} + +func (m *multiLibrary) Libraries() (borges.LibraryIterator, error) { + return m.lib.Libraries() +} + +func recursiveCopy( + dst string, + dstFS billy.Filesystem, + src string, + srcFS billy.Filesystem, +) error { + stat, err := srcFS.Stat(src) + if err != nil { + return err + } + + if stat.IsDir() { + err = dstFS.MkdirAll(dst, stat.Mode()) + if err != nil { + return err + } + + files, err := srcFS.ReadDir(src) + if err != nil { + return err + } + + for _, file := range files { + srcPath := filepath.Join(src, file.Name()) + dstPath := filepath.Join(dst, file.Name()) + + err = recursiveCopy(dstPath, dstFS, srcPath, srcFS) + if err != nil { + return err + } + } + } else { + err = copyFile(dst, dstFS, src, srcFS, stat.Mode()) + if err != nil { + return err + } + } + + return nil +} + +func copyFile( + dst string, + dstFS billy.Filesystem, + src string, + srcFS billy.Filesystem, + mode os.FileMode, +) error { + _, err := srcFS.Stat(src) + if err != nil { + return err + } + + fo, err := srcFS.Open(src) + if err != nil { + return err + } + defer fo.Close() + + fd, err := dstFS.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + defer fd.Close() + + _, err = io.Copy(fd, fo) + if err != nil { + _ = fd.Close() + _ = dstFS.Remove(dst) + return err + } + + return nil +} diff --git a/database_test.go b/database_test.go index 85ead7ddf..3a7f4111d 100644 --- a/database_test.go +++ b/database_test.go @@ -4,6 +4,7 @@ import ( "sort" "testing" + "github.com/src-d/go-borges/libraries" "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" ) @@ -13,7 +14,8 @@ const testDBName = "foo" func TestDatabase_Tables(t *testing.T) { require := require.New(t) - db := getDB(t, testDBName, NewRepositoryPool(0)) + lib := libraries.New(libraries.Options{}) + db := getDB(t, testDBName, NewRepositoryPool(0, lib)) tables := db.Tables() var tableNames []string @@ -43,7 +45,8 @@ func TestDatabase_Tables(t *testing.T) { func TestDatabase_Name(t *testing.T) { require := require.New(t) - db := getDB(t, testDBName, NewRepositoryPool(0)) + lib := libraries.New(libraries.Options{}) + db := getDB(t, testDBName, NewRepositoryPool(0, lib)) require.Equal(testDBName, db.Name()) } diff --git a/docs/using-gitbase/configuration.md b/docs/using-gitbase/configuration.md index 1526264cc..3d076219d 100644 --- a/docs/using-gitbase/configuration.md +++ b/docs/using-gitbase/configuration.md @@ -86,37 +86,35 @@ Help Options: [server command options] --db= Database name (default: gitbase) - -d, --directories= Path where the git repositories are located - (standard and siva), multiple directories can - be defined. Accepts globs. - --depth= load repositories looking at less than - nested subdirectories. (default: 1000) - --host= Host where the server is going to listen - (default: localhost) - -p, --port= Port where the server is going to listen - (default: 3306) + -d, --directories= Path where standard git repositories are located, + multiple directories can be defined. + --format=[git|siva] Library format (default: git) + --bucket= Bucketing level to use with siva libraries (default: 2) + --bare Sets the library to use bare git repositories, used + only with git format libraries + --non-rooted Disables treating siva files as rooted repositories + --host= Host where the server is going to listen (default: + localhost) + -p, --port= Port where the server is going to listen (default: + 3306) -u, --user= User name used for connection (default: root) -P, --password= Password used for connection - -U, --user-file= JSON file with credentials list - [$GITBASE_USER_FILE] + -U, --user-file= JSON file with credentials list [$GITBASE_USER_FILE] -t, --timeout= Timeout in seconds used for connections [$GITBASE_CONNECTION_TIMEOUT] - -i, --index= Directory where the gitbase indexes - information will be persisted. (default: - /var/lib/gitbase/index) [$GITBASE_INDEX_DIR] + -i, --index= Directory where the gitbase indexes information will + be persisted. (default: /var/lib/gitbase/index) + [$GITBASE_INDEX_DIR] --cache= Object cache size in megabytes (default: 512) [$GITBASE_CACHESIZE_MB] - --parallelism= Maximum number of parallel threads per table. - By default, it's the number of CPU cores. 0 - means default, 1 means disabled. + --parallelism= Maximum number of parallel threads per table. By + default, it's the number of CPU cores. 0 means + default, 1 means disabled. --no-squash Disables the table squashing. --trace Enables jaeger tracing [$GITBASE_TRACE] - -r, --readonly Only allow read queries. This disables - creating and deleting indexes as well. Cannot - be used with --user-file. [$GITBASE_READONLY] - --no-git disable the load of git standard repositories. - --no-siva disable the load of siva files. + -r, --readonly Only allow read queries. This disables creating and + deleting indexes as well. Cannot be used with + --user-file. [$GITBASE_READONLY] -v Activates the verbose mode - --log-level=[info|debug|warning|error|fatal] logging level (default: info) - [$GITBASE_LOG_LEVEL] + --log-level=[info|debug|warning|error|fatal] logging level (default: info) [$GITBASE_LOG_LEVEL] ``` \ No newline at end of file diff --git a/docs/using-gitbase/getting-started.md b/docs/using-gitbase/getting-started.md index 502415e99..6e92ec549 100644 --- a/docs/using-gitbase/getting-started.md +++ b/docs/using-gitbase/getting-started.md @@ -116,3 +116,27 @@ As of MySQL 8.0 [the default authentication method is `caching_sha2_password`](h ``` mysql -q -u root -h 127.0.0.1 --default-auth=mysql_native_password ``` + +## Library format specification + +By default the directories added to github should contain normal git repositories. If the format of the repositories is different you have two ways to specify it. + +If all the directories are in the same format you can set it globally with these parameters: + +* `--format`: it can be either `git` for filesystem repositories or `siva` for siva archives +* `--bare`: specifies that git archives are bare, can only be used with `git` format +* `--bucket`: sets the number of characters to use for bucketing, used with `siva` libraries +* `--non-rooted`: disables rooted repositories management in `siva` libraries + +If you are mixing formats you can specify each directory as a `file://` URL with these parameters: + +* `format`: can be `git` or `siva` +* `bare`: `true` or `false` +* `bucket`: the characters to use for directory bucketing +* `rooted`: `true` or `false` + +For example: + +``` +-d 'file:///path/to/git?format=git&bare=true' -d 'file:///path/to/sivas?format=siva&rooted=false&bucket=0' +``` diff --git a/files.go b/files.go index 0824f7973..85ee2e821 100644 --- a/files.go +++ b/files.go @@ -85,7 +85,7 @@ func (r *filesTable) PartitionRows( return nil, err } - if len(repos) > 0 && !stringContains(repos, repo.ID) { + if len(repos) > 0 && !stringContains(repos, repo.ID()) { return noRows, nil } @@ -288,7 +288,7 @@ func (i *filesRowIter) Next() (sql.Row, error) { continue } - return fileToRow(i.repo.ID, i.treeHash, f, i.readContent) + return fileToRow(i.repo.ID(), i.treeHash, f, i.readContent) } } @@ -414,8 +414,7 @@ type filesKeyValueIter struct { } func newFilesKeyValueIter(pool *RepositoryPool, repo *Repository, columns []string) (sql.IndexKeyValueIter, error) { - r := pool.repositories[repo.ID] - idx, err := newRepositoryIndex(r) + idx, err := newRepositoryIndex(repo) if err != nil { return nil, err } @@ -475,7 +474,7 @@ func (i *filesKeyValueIter) Next() ([]interface{}, []byte, error) { } key, err := encodeIndexKey(&fileIndexKey{ - Repository: i.repo.ID, + Repository: i.repo.ID(), Packfile: packfile.String(), Hash: hash, Offset: offset, @@ -487,7 +486,7 @@ func (i *filesKeyValueIter) Next() ([]interface{}, []byte, error) { return nil, nil, err } - row, err := fileToRow(i.repo.ID, i.commit.TreeHash, f, stringContains(i.columns, "blob_content")) + row, err := fileToRow(i.repo.ID(), i.commit.TreeHash, f, stringContains(i.columns, "blob_content")) if err != nil { return nil, nil, err } diff --git a/files_test.go b/files_test.go index 79ff3236b..998238e3e 100644 --- a/files_test.go +++ b/files_test.go @@ -3,10 +3,10 @@ package gitbase import ( "testing" - "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing" ) func TestFilesRowIter(t *testing.T) { diff --git a/fs_error_test.go b/fs_error_test.go index fe4afb3f9..b5d2524e0 100644 --- a/fs_error_test.go +++ b/fs_error_test.go @@ -3,19 +3,20 @@ package gitbase import ( "context" "fmt" + "io/ioutil" "os" "path/filepath" "testing" "time" "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - billy "gopkg.in/src-d/go-billy.v4" + "github.com/src-d/go-borges" + "github.com/src-d/go-borges/plain" fixtures "github.com/src-d/go-git-fixtures" - git "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/plumbing/cache" - "gopkg.in/src-d/go-git.v4/storage/filesystem" "github.com/src-d/go-mysql-server/sql" + "github.com/stretchr/testify/require" + billy "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4/osfs" ) func TestFSErrorTables(t *testing.T) { @@ -54,20 +55,49 @@ func setupErrorRepos(t *testing.T) (*sql.Context, CleanupFunc) { fixture := fixtures.ByTag("worktree").One() baseFS := fixture.Worktree() + tmpDir, err := ioutil.TempDir("", "gitbase") + require.NoError(err) - pool := NewRepositoryPool(cache.DefaultMaxSize) + var rootFS billy.Filesystem = osfs.New(tmpDir) - fs, err := brokenFS(brokenPackfile, baseFS) + lib, pool, err := newMultiPool() require.NoError(err) - pool.Add(billyRepo("packfile", fs)) - fs, err = brokenFS(brokenIndex, baseFS) - require.NoError(err) - pool.Add(billyRepo("index", fs)) + repos := []struct { + name string + t brokenType + }{ + { + name: "packfile", + t: brokenPackfile, + }, + { + name: "index", + t: brokenIndex, + }, + { + name: "ok", + t: brokenNone, + }, + } - fs, err = brokenFS(0, baseFS) - require.NoError(err) - pool.Add(billyRepo("ok", fs)) + var fs billy.Filesystem + for _, repo := range repos { + err = rootFS.MkdirAll(repo.name, 0777) + require.NoError(err) + fs, err = rootFS.Chroot(repo.name) + require.NoError(err) + err = recursiveCopy(repo.name, fs, ".git", baseFS) + require.NoError(err) + fs, err = brokenFS(repo.t, fs) + require.NoError(err) + + loc, err := plain.NewLocation(borges.LocationID(repo.name), fs, &plain.LocationOptions{ + Bare: true, + }) + require.NoError(err) + lib.plain.AddLocation(loc) + } session := NewSession(pool, WithSkipGitErrors(true)) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) @@ -75,6 +105,7 @@ func setupErrorRepos(t *testing.T) (*sql.Context, CleanupFunc) { cleanup := func() { t.Helper() require.NoError(fixtures.Clean()) + require.NoError(os.RemoveAll(tmpDir)) } return ctx, cleanup @@ -84,16 +115,11 @@ func brokenFS( brokenType brokenType, fs billy.Filesystem, ) (billy.Filesystem, error) { - dotFS, err := fs.Chroot(".git") - if err != nil { - return nil, err - } - var brokenFS billy.Filesystem if brokenType == brokenNone { - brokenFS = dotFS + brokenFS = fs } else { - brokenFS = NewBrokenFS(brokenType, dotFS) + brokenFS = NewBrokenFS(brokenType, fs) } return brokenFS, nil @@ -116,43 +142,6 @@ func testTable(t *testing.T, tableName string, number int) { } } -type billyRepository struct { - id string - fs billy.Filesystem - cache cache.Object -} - -func billyRepo(id string, fs billy.Filesystem) repository { - return &billyRepository{id, fs, cache.NewObjectLRUDefault()} -} - -func (r *billyRepository) ID() string { - return r.id -} - -func (r *billyRepository) Repo() (*Repository, error) { - storage := filesystem.NewStorage(r.fs, r.cache) - - repo, err := git.Open(storage, r.fs) - if err != nil { - return nil, err - } - - return NewRepository(r.id, repo, nil), nil -} - -func (r *billyRepository) FS() (billy.Filesystem, error) { - return r.fs, nil -} - -func (r *billyRepository) Path() string { - return r.id -} - -func (r *billyRepository) Cache() cache.Object { - return r.cache -} - type brokenType uint64 const ( @@ -225,6 +214,11 @@ func (fs *BrokenFS) ReadDir(path string) ([]os.FileInfo, error) { return files, err } +func (fs *BrokenFS) Stat(path string) (os.FileInfo, error) { + stat, err := fs.Filesystem.Stat(path) + return stat, err +} + type BrokenFile struct { billy.File count int diff --git a/go.mod b/go.mod index 3f8967af3..5d7a82995 100644 --- a/go.mod +++ b/go.mod @@ -5,15 +5,16 @@ go 1.12 require ( github.com/bblfsh/go-client/v4 v4.1.0 github.com/bblfsh/sdk/v3 v3.1.0 - github.com/emirpasic/gods v1.12.0 // indirect + github.com/gliderlabs/ssh v0.2.0 // indirect github.com/go-sql-driver/mysql v1.4.1 - github.com/hashicorp/golang-lru v0.5.0 + github.com/hashicorp/golang-lru v0.5.1 github.com/hhatto/gocloc v0.3.0 github.com/jessevdk/go-flags v1.4.0 github.com/miekg/dns v1.1.1 // indirect github.com/opentracing/opentracing-go v1.1.0 github.com/sirupsen/logrus v1.3.0 github.com/src-d/enry/v2 v2.0.0 + github.com/src-d/go-borges v0.0.0-20190619084057-d02cf3fd6581 github.com/src-d/go-git v4.7.0+incompatible github.com/src-d/go-git-fixtures v3.5.1-0.20190605154830-57f3972b0248+incompatible github.com/src-d/go-mysql-server v0.4.1-0.20190619104848-eaab1795353a @@ -22,13 +23,16 @@ require ( github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.0.0+incompatible // indirect go.uber.org/atomic v1.4.0 // indirect + golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 // indirect + golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect + golang.org/x/sys v0.0.0-20190618155005-516e3c20635f // indirect google.golang.org/grpc v1.20.1 - gopkg.in/src-d/go-billy-siva.v4 v4.2.2 + gopkg.in/src-d/go-billy-siva.v4 v4.5.1 gopkg.in/src-d/go-billy.v4 v4.3.0 gopkg.in/src-d/go-errors.v1 v1.0.0 gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 - gopkg.in/src-d/go-git.v4 v4.11.0 - gopkg.in/src-d/go-siva.v1 v1.3.0 // indirect + gopkg.in/src-d/go-git.v4 v4.12.0 gopkg.in/yaml.v2 v2.2.2 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index 95cb9e354..16b4e0d29 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/antchfx/xpath v0.0.0-20190319080838-ce1d48779e67/go.mod h1:Yee4kTMuNi github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/bblfsh/go-client/v4 v4.1.0 h1:z9S5GrUSB0uiipbWauc5fV6D/R0LNBi65dhfP5uh3kw= github.com/bblfsh/go-client/v4 v4.1.0/go.mod h1:UUAG7jrxSr7WHlFL/U8den1kLHfysZWJ9jA1Y0IiQ+0= github.com/bblfsh/sdk/v3 v3.1.0 h1:V+4DqGlqbxiAEAU0XlpTjjbq/g0ljfiFoh9jsIkfbtU= @@ -52,11 +54,17 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/gliderlabs/ssh v0.1.3/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/gliderlabs/ssh v0.1.4 h1:5N8AYXpaQAPy0L7linKa5aI+WRfyYagAhjksVzxh+mI= +github.com/gliderlabs/ssh v0.1.4/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/gliderlabs/ssh v0.2.0 h1:x0lYvhr3g30Vo8ISP+XgrP1KoC0/BiUUTY/HsosqSS4= +github.com/gliderlabs/ssh v0.2.0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= @@ -79,6 +87,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github v15.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -103,6 +113,8 @@ github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCS github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= @@ -126,10 +138,13 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mcuadros/go-lookup v0.0.0-20171110082742-5650f26be767 h1:BrhJNdEFWGuiJk/3/SwsG5Rex3zjFxYsDi2bpd7382Y= github.com/mcuadros/go-lookup v0.0.0-20171110082742-5650f26be767/go.mod h1:ct+byCpkFokm4J0tiuAvB8cf2ttm6GcCe89Yr25nGKg= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -137,6 +152,8 @@ github.com/miekg/dns v1.1.1 h1:DVkblRdiScEnEr0LR9nTnEQqHYycjkXW9bOjd+2EL2o= github.com/miekg/dns v1.1.1/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -202,6 +219,8 @@ github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/src-d/go-git v4.7.0+incompatible h1:IYSSnbAHeKmsfbQFi9ozbid+KNh0bKjlorMfQehQbcE= github.com/src-d/go-git v4.7.0+incompatible/go.mod h1:1bQciz+hn0jzPQNsYj0hDFZHLJBdV7gXE2mWhC7EkFk= +github.com/src-d/go-borges v0.0.0-20190619084057-d02cf3fd6581 h1:ZOJFuELokTVFnwpgD0tj9PMTXiBwutmGoH2WCaG5dts= +github.com/src-d/go-borges v0.0.0-20190619084057-d02cf3fd6581/go.mod h1:Myl/zHrk3iT/I5T08RTBpuGzchucytSsi6p7KzM2lOA= github.com/src-d/go-git-fixtures v3.5.1-0.20190605154830-57f3972b0248+incompatible h1:A5bKevhs9C//Nh8QV0J+1KphEaIa25cDe1DTs/yPxDI= github.com/src-d/go-git-fixtures v3.5.1-0.20190605154830-57f3972b0248+incompatible/go.mod h1:XcIQp7L+k0pgfTqfbaTKj3kxlBv8kYOKZ/tKNXbZFLg= github.com/src-d/go-mysql-server v0.4.1-0.20190619104848-eaab1795353a h1:tAgjKO6ynjJasm6wTSjmFJIf/lh/Rk3dPVCUnOXAWEQ= @@ -230,9 +249,12 @@ github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/Aaua github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro= github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= +github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= +github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -244,8 +266,14 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 h1:IcSOAf4PyMp3U3XbIEj1/xJ2BjNN2jWv7JoyOsMxXUU= +golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -259,8 +287,14 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190227022144-312bce6e941f h1:tbtX/qtlxzhZjgQue/7u7ygFwDEckd+DmS5+t8FgeKE= golang.org/x/net v0.0.0-20190227022144-312bce6e941f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190520210107-018c4d40a106 h1:EZofHp/BzEf3j39/+7CX1JvH0WaPG+ikBrqAdAPf+GM= golang.org/x/net v0.0.0-20190520210107-018c4d40a106/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190607181551-461777fb6f67 h1:rJJxsykSlULwd2P2+pg/rtnwN2FrWp4IuCxOSyS0V00= +golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -268,6 +302,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -276,8 +312,15 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190609082536-301114b31cce h1:CQakrGkKbydnUmt7cFIlmQ4lNQiqdTPt6xzXij4nYCc= +golang.org/x/sys v0.0.0-20190609082536-301114b31cce/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190618155005-516e3c20635f h1:dHNZYIYdq2QuU6w73vZ/DzesPbVlZVYZTtTZmrnsbQ8= +golang.org/x/sys v0.0.0-20190618155005-516e3c20635f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -310,8 +353,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/src-d/go-billy-siva.v4 v4.2.2 h1:HvklDMblrg/Zknu4tAFJayg34QUOvuojXjMQGqA2FtM= -gopkg.in/src-d/go-billy-siva.v4 v4.2.2/go.mod h1:4wKeCzOCSsdyFeM5+58M6ObU6FM+lZT12p7zm7A+9n0= +gopkg.in/src-d/go-billy-siva.v4 v4.5.1 h1:+UdpGGmJjANhXwg6TCcTVbACUqsbtX19QvJ9AdeX4ts= +gopkg.in/src-d/go-billy-siva.v4 v4.5.1/go.mod h1:4wKeCzOCSsdyFeM5+58M6ObU6FM+lZT12p7zm7A+9n0= gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= @@ -323,8 +366,10 @@ gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOA gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= gopkg.in/src-d/go-git.v4 v4.11.0 h1:cJwWgJ0DXifrNrXM6RGN1Y2yR60Rr1zQ9Q5DX5S9qgU= gopkg.in/src-d/go-git.v4 v4.11.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk= -gopkg.in/src-d/go-siva.v1 v1.3.0 h1:ARW6NvQNYIRLr0Hrftiyb7vWW9p0gWk9KchMz2hiH0I= -gopkg.in/src-d/go-siva.v1 v1.3.0/go.mod h1:tk1jnIXawd/PTlRNWdr5V5lC0PttNJmu1fv7wt7IZlw= +gopkg.in/src-d/go-git.v4 v4.12.0 h1:CKgvBCJCcdfNnyXPYI4Cp8PaDDAmAPEN0CtfEdEAbd8= +gopkg.in/src-d/go-git.v4 v4.12.0/go.mod h1:zjlNnzc1Wjn43v3Mtii7RVxiReNP0fIu9npcXKzuNp4= +gopkg.in/src-d/go-siva.v1 v1.5.0 h1:WowvbZTlz0SPoV7WNCGktPSi2yRK78HPyXl7wYqDeHE= +gopkg.in/src-d/go-siva.v1 v1.5.0/go.mod h1:tk1jnIXawd/PTlRNWdr5V5lC0PttNJmu1fv7wt7IZlw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE= gopkg.in/toqueteos/substring.v1 v1.0.2/go.mod h1:Eb2Z1UYehlVK8LYW2WBVR2rwbujsz3aX8XDrM1vbNew= diff --git a/index_test.go b/index_test.go index 4d89bb24a..22208ef81 100644 --- a/index_test.go +++ b/index_test.go @@ -5,9 +5,9 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" "gopkg.in/src-d/go-git.v4/plumbing" - "github.com/src-d/go-mysql-server/sql" ) func assertEncodeKey(t *testing.T, key indexKey) []byte { diff --git a/init.sh b/init.sh index 54e0967fd..97aa35635 100644 --- a/init.sh +++ b/init.sh @@ -1,5 +1,9 @@ #!/bin/sh +if [ -n "$SIVA" ]; then + SIVA_ARGS="--format siva --bucket 2" +fi + cat <> "$HOME/.my.cnf" [client] user=${GITBASE_USER} @@ -11,4 +15,5 @@ EOT --port=3306 \ --user="$GITBASE_USER" \ --password="$GITBASE_PASSWORD" \ - --directories="$GITBASE_REPOS" \ No newline at end of file + --directories="$GITBASE_REPOS" \ + $SIVA_ARGS \ No newline at end of file diff --git a/integration_test.go b/integration_test.go index a4b121d2f..16f6671c8 100644 --- a/integration_test.go +++ b/integration_test.go @@ -12,6 +12,8 @@ import ( "github.com/src-d/gitbase/cmd/gitbase/command" "github.com/src-d/gitbase/internal/rule" + "github.com/src-d/go-borges/plain" + "github.com/src-d/go-borges/siva" "github.com/src-d/gitbase" "github.com/src-d/gitbase/internal/function" @@ -22,6 +24,7 @@ import ( "github.com/src-d/go-mysql-server/sql/analyzer" "github.com/src-d/go-mysql-server/sql/index/pilosa" "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-git.v4/plumbing/cache" ) @@ -31,10 +34,24 @@ func TestIntegration(t *testing.T) { }() path := fixtures.ByTag("worktree").One().Worktree().Root() + pathLib := path + "-lib" + pathRepo := filepath.Join(pathLib, "worktree") - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) - require.NoError(t, pool.AddGitWithID("worktree", path)) + err := os.MkdirAll(pathLib, 0777) + require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(pathLib)) + }() + + err = os.Rename(path, pathRepo) + require.NoError(t, err) + + lib := plain.NewLibrary("plain") + loc, err := plain.NewLocation("location", osfs.New(pathLib), nil) + require.NoError(t, err) + lib.AddLocation(loc) + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) engine := newBaseEngine(pool) testCases := []struct { @@ -612,27 +629,17 @@ func queryResults( func TestMissingHeadRefs(t *testing.T) { require := require.New(t) - path := filepath.Join( - os.Getenv("GOPATH"), - "src", "github.com", "src-d", "gitbase", - "_testdata", - ) - - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) - require.NoError( - filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } + cwd, err := os.Getwd() + require.NoError(err) - if gitbase.IsSivaFile(path) { - require.NoError(pool.AddSivaFile(path)) - } + path := filepath.Join(cwd, "_testdata") - return nil - }), - ) + lib, err := siva.NewLibrary("siva", osfs.New(path), siva.LibraryOptions{ + RootedRepo: true, + }) + require.NoError(err) + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) engine := newBaseEngine(pool) session := gitbase.NewSession(pool) @@ -642,7 +649,7 @@ func TestMissingHeadRefs(t *testing.T) { rows, err := sql.RowIterToRows(iter) require.NoError(err) - require.Len(rows, 56) + require.Len(rows, 57) } func BenchmarkQueries(b *testing.B) { @@ -988,13 +995,29 @@ func deleteIndex( func setup(t testing.TB) (*sqle.Engine, *gitbase.RepositoryPool, func()) { t.Helper() + + // create library directory and move repo inside + + path := fixtures.ByTag("worktree").One().Worktree().Root() + pathLib := path + "-lib" + pathRepo := filepath.Join(pathLib, "worktree") + + err := os.MkdirAll(pathLib, 0777) + require.NoError(t, err) + + err = os.Rename(path, pathRepo) + require.NoError(t, err) + + lib := plain.NewLibrary("plain") + loc, err := plain.NewLocation("location", osfs.New(pathLib), nil) + require.NoError(t, err) + lib.AddLocation(loc) + + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) + cleanup := func() { require.NoError(t, fixtures.Clean()) - } - - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) - for _, f := range fixtures.ByTag("worktree") { - pool.AddGitWithID("worktree", f.Worktree().Root()) + require.NoError(t, os.RemoveAll(pathLib)) } return newBaseEngine(pool), pool, cleanup diff --git a/internal/function/commit_file_stats_test.go b/internal/function/commit_file_stats_test.go index 0cc3d5e9e..2a3875dd4 100644 --- a/internal/function/commit_file_stats_test.go +++ b/internal/function/commit_file_stats_test.go @@ -10,20 +10,11 @@ import ( "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" - fixtures "gopkg.in/src-d/go-git-fixtures.v3" - "gopkg.in/src-d/go-git.v4/plumbing/cache" ) func TestCommitFileStats(t *testing.T) { - require.NoError(t, fixtures.Init()) - defer func() { - require.NoError(t, fixtures.Clean()) - }() - - path := fixtures.ByTag("worktree").One().Worktree().Root() - - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) - require.NoError(t, pool.AddGitWithID("worktree", path)) + pool, cleanup := setupPool(t) + defer cleanup() session := gitbase.NewSession(pool) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) diff --git a/internal/function/commit_stats_test.go b/internal/function/commit_stats_test.go index af99a5650..63e9f269c 100644 --- a/internal/function/commit_stats_test.go +++ b/internal/function/commit_stats_test.go @@ -2,28 +2,25 @@ package function import ( "context" + "os" + "path/filepath" "testing" "github.com/src-d/gitbase" "github.com/src-d/gitbase/internal/commitstats" + "github.com/src-d/go-borges/plain" "github.com/stretchr/testify/require" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "gopkg.in/src-d/go-billy.v4/osfs" fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing/cache" ) func TestCommitStatsEval(t *testing.T) { - require.NoError(t, fixtures.Init()) - defer func() { - require.NoError(t, fixtures.Clean()) - }() - - path := fixtures.ByTag("worktree").One().Worktree().Root() - - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) - require.NoError(t, pool.AddGitWithID("worktree", path)) + pool, cleanup := setupPool(t) + defer cleanup() session := gitbase.NewSession(pool) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) @@ -86,3 +83,32 @@ func TestCommitStatsEval(t *testing.T) { }) } } + +func setupPool(t *testing.T) (*gitbase.RepositoryPool, func()) { + t.Helper() + require.NoError(t, fixtures.Init()) + + path := fixtures.ByTag("worktree").One().Worktree().Root() + pathLib := path + "-lib" + pathRepo := filepath.Join(pathLib, "worktree") + + cleanup := func() { + require.NoError(t, fixtures.Clean()) + require.NoError(t, os.RemoveAll(pathLib)) + } + + err := os.MkdirAll(pathLib, 0777) + require.NoError(t, err) + + err = os.Rename(path, pathRepo) + require.NoError(t, err) + + lib := plain.NewLibrary("plain") + loc, err := plain.NewLocation("location", osfs.New(pathLib), nil) + require.NoError(t, err) + lib.AddLocation(loc) + + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) + + return pool, cleanup +} diff --git a/internal/function/uast_test.go b/internal/function/uast_test.go index 0c2ef06de..77c68690c 100644 --- a/internal/function/uast_test.go +++ b/internal/function/uast_test.go @@ -3,19 +3,23 @@ package function import ( "context" "encoding/json" + "os" + "path/filepath" "reflect" "testing" "github.com/bblfsh/sdk/v3/uast" - "github.com/src-d/gitbase" - "github.com/stretchr/testify/require" bblfsh "github.com/bblfsh/go-client/v4" "github.com/bblfsh/sdk/v3/uast/nodes" + "github.com/src-d/gitbase" + "github.com/src-d/go-borges/plain" fixtures "github.com/src-d/go-git-fixtures" - "gopkg.in/src-d/go-git.v4/plumbing/cache" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git.v4/plumbing/cache" ) const testCode = ` @@ -419,10 +423,29 @@ func bblfshFixtures( func setup(t *testing.T) (*sql.Context, func()) { t.Helper() - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) - for _, f := range fixtures.ByTag("worktree") { - pool.AddGit(f.Worktree().Root()) - } + // pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) + // for _, f := range fixtures.ByTag("worktree") { + // pool.AddGit(f.Worktree().Root()) + // } + + // create library directory and move repo inside + + path := fixtures.ByTag("worktree").One().Worktree().Root() + pathLib := path + "-lib" + pathRepo := filepath.Join(pathLib, "worktree") + + err := os.MkdirAll(pathLib, 0777) + require.NoError(t, err) + + err = os.Rename(path, pathRepo) + require.NoError(t, err) + + lib := plain.NewLibrary("plain") + loc, err := plain.NewLocation("location", osfs.New(pathLib), nil) + require.NoError(t, err) + lib.AddLocation(loc) + + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) session := gitbase.NewSession(pool) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) diff --git a/internal/rule/squashjoins_test.go b/internal/rule/squashjoins_test.go index 800287e42..c5f768ab8 100644 --- a/internal/rule/squashjoins_test.go +++ b/internal/rule/squashjoins_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/src-d/gitbase" + "github.com/src-d/go-borges/libraries" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/analyzer" "github.com/src-d/go-mysql-server/sql/expression" @@ -17,9 +18,11 @@ import ( func TestAnalyzeSquashJoinsExchange(t *testing.T) { require := require.New(t) + lib := libraries.New(libraries.Options{}) + catalog := sql.NewCatalog() catalog.AddDatabase( - gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0)), + gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)), ) a := analyzer.NewBuilder(catalog). WithParallelism(2). @@ -54,9 +57,11 @@ func TestAnalyzeSquashJoinsExchange(t *testing.T) { func TestAnalyzeSquashNaturalJoins(t *testing.T) { require := require.New(t) + lib := libraries.New(libraries.Options{}) + catalog := sql.NewCatalog() catalog.AddDatabase( - gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0)), + gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)), ) a := analyzer.NewBuilder(catalog). WithParallelism(2). @@ -2283,7 +2288,8 @@ func TestIsJoinLeafSquashable(t *testing.T) { } func TestOrderedTableNames(t *testing.T) { - tables := gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0)).Tables() + lib := libraries.New(libraries.Options{}) + tables := gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)).Tables() input := []sql.Table{ tables[gitbase.BlobsTableName], @@ -2824,5 +2830,6 @@ func (l dummyLookup) Indexes() []string { } func gitbaseTables() map[string]sql.Table { - return gitbase.NewDatabase("", gitbase.NewRepositoryPool(0)).Tables() + lib := libraries.New(libraries.Options{}) + return gitbase.NewDatabase("", gitbase.NewRepositoryPool(0, lib)).Tables() } diff --git a/packfiles.go b/packfiles.go index 3bfb7db90..f425e01c3 100644 --- a/packfiles.go +++ b/packfiles.go @@ -11,7 +11,7 @@ import ( "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" - "gopkg.in/src-d/go-billy-siva.v4" + sivafs "gopkg.in/src-d/go-billy-siva.v4" "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" @@ -45,7 +45,7 @@ type repositoryIndex struct { closeFunc func() } -func newRepositoryIndex(repo repository) (*repositoryIndex, error) { +func newRepositoryIndex(repo *Repository) (*repositoryIndex, error) { fs, err := repo.FS() if err != nil { return nil, err @@ -151,7 +151,7 @@ func findDotGit(fs billy.Filesystem) (billy.Filesystem, error) { return fs, nil } -func getUnpackedObject(repo repository, hash plumbing.Hash) (o object.Object, err error) { +func getUnpackedObject(repo *Repository, hash plumbing.Hash) (o object.Object, err error) { fs, err := repo.FS() if err != nil { return nil, err @@ -212,7 +212,7 @@ type repoObjectDecoder struct { } func newRepoObjectDecoder( - repo repository, + repo *Repository, hash plumbing.Hash, ) (*repoObjectDecoder, error) { fs, err := repo.FS() @@ -290,6 +290,11 @@ func (d *objectDecoder) decode( offset int64, hash plumbing.Hash, ) (object.Object, error) { + repo, err := d.pool.GetRepo(repository) + if err != nil { + return nil, err + } + if offset >= 0 { if d.decoder == nil || !d.decoder.equals(repository, packfile) { if d.decoder != nil { @@ -299,7 +304,7 @@ func (d *objectDecoder) decode( } var err error - d.decoder, err = newRepoObjectDecoder(d.pool.repositories[repository], packfile) + d.decoder, err = newRepoObjectDecoder(repo, packfile) if err != nil { return nil, err } @@ -308,7 +313,7 @@ func (d *objectDecoder) decode( return d.decoder.get(offset) } - return getUnpackedObject(d.pool.repositories[repository], hash) + return getUnpackedObject(repo, hash) } func (d *objectDecoder) Close() error { diff --git a/packfiles_test.go b/packfiles_test.go index ed4c2d5f0..a9d5968a6 100644 --- a/packfiles_test.go +++ b/packfiles_test.go @@ -1,20 +1,35 @@ package gitbase import ( + "os" "path/filepath" "testing" "github.com/stretchr/testify/require" "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/cache" ) -var testSivaFilePath = filepath.Join("_testdata", "fff7062de8474d10a67d417ccea87ba6f58ca81d.siva") +var ( + testSivaFilePath = filepath.Join("_testdata", "fff7062de8474d10a67d417ccea87ba6f58ca81d.siva") + testSivaRepoID = "015dcc49-9049-b00c-ba72-b6f5fa98cbe7" +) func TestRepositoryPackfiles(t *testing.T) { require := require.New(t) - repo := sivaRepo("siva", testSivaFilePath, cache.NewObjectLRUDefault()) + lib, pool, err := newMultiPool() + require.NoError(err) + + cwd, err := os.Getwd() + require.NoError(err) + p := filepath.Join(cwd, testSivaFilePath) + + err = lib.AddSiva(p, nil) + require.NoError(err) + + repo, err := pool.GetRepo(testSivaRepoID) + require.NoError(err) + f, err := repo.FS() require.NoError(err) @@ -22,14 +37,27 @@ func TestRepositoryPackfiles(t *testing.T) { require.NoError(err) require.Equal([]plumbing.Hash{ - plumbing.NewHash("5d2ce6a45cb07803f9b0c8040e730f5715fc7144"), plumbing.NewHash("433e5205f6e26099e7d34ba5e5306f69e4cef12b"), + plumbing.NewHash("5d2ce6a45cb07803f9b0c8040e730f5715fc7144"), }, packfiles) require.NotNil(fs) } func TestRepositoryIndex(t *testing.T) { - idx, err := newRepositoryIndex(sivaRepo("siva", testSivaFilePath, cache.NewObjectLRUDefault())) + lib, pool, err := newMultiPool() + require.NoError(t, err) + + cwd, err := os.Getwd() + require.NoError(t, err) + p := filepath.Join(cwd, testSivaFilePath) + + err = lib.AddSiva(p, nil) + require.NoError(t, err) + + repo, err := pool.GetRepo(testSivaRepoID) + require.NoError(t, err) + + idx, err := newRepositoryIndex(repo) require.NoError(t, err) testCases := []struct { diff --git a/partition.go b/partition.go index 323f94159..d3e0577d2 100644 --- a/partition.go +++ b/partition.go @@ -3,8 +3,9 @@ package gitbase import ( "io" - errors "gopkg.in/src-d/go-errors.v1" + "github.com/src-d/go-borges" "github.com/src-d/go-mysql-server/sql" + errors "gopkg.in/src-d/go-errors.v1" ) // partitioned is an embeddable helper that contains the methods for a table @@ -21,7 +22,22 @@ func (partitioned) PartitionCount(ctx *sql.Context) (int64, error) { return 0, err } - return int64(len(s.Pool.repositories)), nil + it, err := s.Pool.RepoIter() + if err != nil { + return 0, err + } + + var count int64 + for { + _, err = it.Next() + if err == io.EOF { + return count, nil + } + if err != nil { + return 0, err + } + count++ + } } // RepositoryPartition represents a partition which is a repository id. @@ -33,8 +49,9 @@ func (p RepositoryPartition) Key() []byte { } type repositoryPartitionIter struct { - repos []string - pos int + repoIter borges.RepositoryIterator + lib borges.Library + skipErrors bool } func newRepositoryPartitionIter(ctx *sql.Context) (sql.PartitionIter, error) { @@ -43,20 +60,42 @@ func newRepositoryPartitionIter(ctx *sql.Context) (sql.PartitionIter, error) { return nil, err } - return &repositoryPartitionIter{repos: s.Pool.idOrder}, nil + it, err := s.Pool.library.Repositories(borges.ReadOnlyMode) + if err != nil { + return nil, err + } + + return &repositoryPartitionIter{ + repoIter: it, + lib: s.Pool.library, + skipErrors: s.SkipGitErrors, + }, nil } func (i *repositoryPartitionIter) Next() (sql.Partition, error) { - if i.pos >= len(i.repos) { - return nil, io.EOF + var r borges.Repository + var err error + for { + r, err = i.repoIter.Next() + if err == nil { + break + } + if err == io.EOF || !i.skipErrors { + return nil, err + } } - i.pos++ - return RepositoryPartition(i.repos[i.pos-1]), nil + // TODO: RepositoryPartition should hold the repository so we don't + // get it twice. + defer r.Close() + return RepositoryPartition(r.ID().String()), nil } func (i *repositoryPartitionIter) Close() error { - i.pos = len(i.repos) + if i.repoIter != nil { + i.repoIter.Close() + } + return nil } diff --git a/ref_commits.go b/ref_commits.go index 1268f26b3..5c71ecc13 100644 --- a/ref_commits.go +++ b/ref_commits.go @@ -88,7 +88,7 @@ func (t *refCommitsTable) PartitionRows( return nil, err } - if len(repos) > 0 && !stringContains(repos, repo.ID) { + if len(repos) > 0 && !stringContains(repos, repo.ID()) { return noRows, nil } @@ -346,7 +346,7 @@ func (i *refCommitsRowIter) next() (sql.Row, error) { } return sql.NewRow( - i.repo.ID, + i.repo.ID(), commit.Hash.String(), i.ref.Name().String(), int64(idx), diff --git a/ref_commits_test.go b/ref_commits_test.go index dd284448a..6568c3c41 100644 --- a/ref_commits_test.go +++ b/ref_commits_test.go @@ -3,10 +3,10 @@ package gitbase import ( "testing" - "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing" ) func TestRefCommitsRowIter(t *testing.T) { diff --git a/references.go b/references.go index dc592f77d..c2ea04291 100644 --- a/references.go +++ b/references.go @@ -290,7 +290,7 @@ func (i *refRowIter) next() (sql.Row, error) { } return sql.NewRow( - i.repo.ID, + i.repo.ID(), "HEAD", o.Hash().String(), ), nil @@ -313,7 +313,7 @@ func (i *refRowIter) next() (sql.Row, error) { continue } - return referenceToRow(i.repo.ID, o), nil + return referenceToRow(i.repo.ID(), o), nil } } diff --git a/references_test.go b/references_test.go index ccd2efcc0..fe99a4d06 100644 --- a/references_test.go +++ b/references_test.go @@ -3,10 +3,10 @@ package gitbase import ( "testing" - "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing" ) func TestReferencesTable(t *testing.T) { diff --git a/remotes.go b/remotes.go index 78ce9cf2e..48125fcef 100644 --- a/remotes.go +++ b/remotes.go @@ -158,7 +158,7 @@ func (i *remotesRowIter) Next() (sql.Row, error) { continue } - row := remoteToRow(i.repo.ID, config, i.urlPos) + row := remoteToRow(i.repo.ID(), config, i.urlPos) i.urlPos++ return row, nil @@ -267,12 +267,12 @@ func (i *remotesKeyValueIter) Next() ([]interface{}, []byte, error) { i.urlPos++ - key, err := encodeIndexKey(&remoteIndexKey{i.repo.ID, i.pos, i.urlPos - 1}) + key, err := encodeIndexKey(&remoteIndexKey{i.repo.ID(), i.pos, i.urlPos - 1}) if err != nil { return nil, nil, err } - row := remoteToRow(i.repo.ID, cfg, i.urlPos-1) + row := remoteToRow(i.repo.ID(), cfg, i.urlPos-1) values, err := rowIndexValues(row, i.columns, RemotesSchema) if err != nil { return nil, nil, err diff --git a/remotes_test.go b/remotes_test.go index 203262e5d..e9a76ca4b 100644 --- a/remotes_test.go +++ b/remotes_test.go @@ -4,6 +4,7 @@ import ( "fmt" "testing" + "github.com/src-d/go-borges" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" @@ -13,15 +14,18 @@ import ( func TestRemotesTable(t *testing.T) { require := require.New(t) - ctx, _, cleanup := setup(t) + ctx, fix, cleanup := setup(t) defer cleanup() table := newRemotesTable(poolFromCtx(t, ctx)) session := ctx.Session.(*Session) pool := session.Pool - repo, err := pool.GetPos(0) + lib := pool.library + + bRepo, err := lib.Get(borges.RepositoryID(fix), borges.RWMode) require.NoError(err) + r := bRepo.R() config := gitconfig.RemoteConfig{ Name: "my_remote", @@ -32,7 +36,13 @@ func TestRemotesTable(t *testing.T) { }, } - _, err = repo.CreateRemote(&config) + _, err = r.CreateRemote(&config) + require.NoError(err) + + err = bRepo.Close() + require.NoError(err) + + repo, err := pool.GetRepo(fix) require.NoError(err) rows, err := tableToRows(ctx, table) @@ -50,7 +60,7 @@ func TestRemotesTable(t *testing.T) { num := urlstring[len(urlstring)-1:] - require.Equal(repo.ID, row[0]) + require.Equal(repo.ID(), row[0]) url := fmt.Sprintf("url%v", num) require.Equal(url, row[2]) // push diff --git a/repositories.go b/repositories.go index 5a4b926aa..352b40b2a 100644 --- a/repositories.go +++ b/repositories.go @@ -146,7 +146,7 @@ func (i *repositoriesRowIter) Next() (sql.Row, error) { } i.visited = true - return sql.NewRow(i.repo.ID), nil + return sql.NewRow(i.repo.ID()), nil } func (i *repositoriesRowIter) Close() error { diff --git a/repositories_test.go b/repositories_test.go index 7f982b22c..e8854f92e 100644 --- a/repositories_test.go +++ b/repositories_test.go @@ -4,10 +4,11 @@ import ( "context" "testing" - "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing/cache" + fixtures "github.com/src-d/go-git-fixtures" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing/cache" ) func TestRepositoriesTable(t *testing.T) { @@ -18,10 +19,15 @@ func TestRepositoriesTable(t *testing.T) { "seven", "eight", "nine", } - pool := NewRepositoryPool(cache.DefaultMaxSize) + lib, err := newMultiLibrary() + require.NoError(err) + + pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() for _, id := range repoIDs { - pool.Add(gitRepo(id, "", pool.cache)) + err = lib.AddPlain(id, path, nil) + require.NoError(err) } session := NewSession(pool) diff --git a/repository_pool.go b/repository_pool.go index b08d53670..1e495ac9a 100644 --- a/repository_pool.go +++ b/repository_pool.go @@ -1,17 +1,13 @@ package gitbase import ( + "fmt" "io" - "io/ioutil" - "os" - "path/filepath" - "sync/atomic" - "gopkg.in/src-d/go-billy-siva.v4" + "github.com/src-d/go-borges" billy "gopkg.in/src-d/go-billy.v4" - "gopkg.in/src-d/go-billy.v4/osfs" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-git.v4" + git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" ) @@ -25,237 +21,68 @@ var ( } ) -// Repository struct holds an initialized repository and its ID type Repository struct { *git.Repository - ID string - // function used to close underlying file descriptors - closeFunc func() + cache cache.Object + repo borges.Repository + lib borges.Library } -// NewRepository creates and initializes a new Repository structure -func NewRepository(id string, repo *git.Repository, closeFunc func()) *Repository { +func NewRepository( + lib borges.Library, + repo borges.Repository, + cache cache.Object, +) *Repository { return &Repository{ - Repository: repo, - ID: id, - closeFunc: closeFunc, + Repository: repo.R(), + lib: lib, + repo: repo, + cache: cache, } } -// Close closes all opened files in the repository. -func (r *Repository) Close() { - if r != nil { - if r.Repository != nil { - f, ok := r.Storer.(*filesystem.Storage) - if ok { - // The only type of error returned is "file already closed" and - // we don't want to do anything with it. - f.Close() - } - } - - if r.closeFunc != nil { - r.closeFunc() - } - } +func (r *Repository) ID() string { + return r.repo.ID().String() } -// NewRepositoryFromPath creates and initializes a new Repository structure -// and initializes a go-git repository -func NewRepositoryFromPath(id, path string, cache cache.Object) (*Repository, error) { - var wt billy.Filesystem - fs := osfs.New(path) - f, err := fs.Stat(git.GitDirName) - if err != nil && !os.IsNotExist(err) { - return nil, err - } - - if f != nil && f.IsDir() { - wt = fs - fs, err = fs.Chroot(git.GitDirName) - if err != nil { - return nil, err - } - } - - sto := filesystem.NewStorageWithOptions(fs, cache, gitStorerOptions) - repo, err := git.Open(sto, wt) - if err != nil { - return nil, err - } - - return NewRepository(id, repo, nil), nil -} - -// NewSivaRepositoryFromPath creates and initializes a new Repository structure -// and initializes a go-git repository backed by a siva file. -func NewSivaRepositoryFromPath(id, path string, cache cache.Object) (*Repository, error) { - localfs := osfs.New(filepath.Dir(path)) - - tmpDir, err := ioutil.TempDir(os.TempDir(), "gitbase-siva") - if err != nil { - return nil, err - } - - tmpfs := osfs.New(tmpDir) - - fs, err := sivafs.NewFilesystem(localfs, filepath.Base(path), tmpfs) - if err != nil { - return nil, err - } - - sto := filesystem.NewStorageWithOptions(fs, cache, gitStorerOptions) - - repo, err := git.Open(sto, nil) - if err != nil { - return nil, err +func (r *Repository) FS() (billy.Filesystem, error) { + fs := r.repo.FS() + if fs == nil { + return nil, fmt.Errorf("filesystem inaccesible") } - closeFunc := func() { fs.Sync() } - - return NewRepository(id, repo, closeFunc), nil + return fs, nil } -type repository interface { - ID() string - Repo() (*Repository, error) - FS() (billy.Filesystem, error) - Path() string - Cache() cache.Object -} - -type gitRepository struct { - id string - path string - cache cache.Object -} - -func gitRepo(id, path string, cache cache.Object) repository { - return &gitRepository{id, path, cache} -} - -func (r *gitRepository) ID() string { - return r.id -} - -func (r *gitRepository) Repo() (*Repository, error) { - return NewRepositoryFromPath(r.id, r.path, r.cache) -} - -func (r *gitRepository) FS() (billy.Filesystem, error) { - return osfs.New(r.path), nil -} - -func (r *gitRepository) Path() string { - return r.path -} - -func (r *gitRepository) Cache() cache.Object { +func (r *Repository) Cache() cache.Object { return r.cache } -type sivaRepository struct { - id string - path string - cache cache.Object -} - -func sivaRepo(id, path string, cache cache.Object) repository { - return &sivaRepository{id, path, cache} -} - -func (r *sivaRepository) ID() string { - return r.id -} - -func (r *sivaRepository) Repo() (*Repository, error) { - return NewSivaRepositoryFromPath(r.id, r.path, r.cache) -} - -func (r *sivaRepository) FS() (billy.Filesystem, error) { - localfs := osfs.New(filepath.Dir(r.path)) - - tmpDir, err := ioutil.TempDir(os.TempDir(), "gitbase-siva") - if err != nil { - return nil, err +func (r *Repository) Close() { + if r != nil && r.repo != nil { + if closer, ok := r.repo.(io.Closer); ok { + closer.Close() + } } - - tmpfs := osfs.New(tmpDir) - - return sivafs.NewFilesystem(localfs, filepath.Base(r.path), tmpfs) -} - -func (r *sivaRepository) Path() string { - return r.path -} - -func (r *sivaRepository) Cache() cache.Object { - return r.cache } // RepositoryPool holds a pool git repository paths and // functionality to open and iterate them. type RepositoryPool struct { - repositories map[string]repository - idOrder []string - cache cache.Object + cache cache.Object + library borges.Library } // NewRepositoryPool initializes a new RepositoryPool with LRU cache. -func NewRepositoryPool(maxCacheSize cache.FileSize) *RepositoryPool { +func NewRepositoryPool( + maxCacheSize cache.FileSize, + lib borges.Library, +) *RepositoryPool { return &RepositoryPool{ - repositories: make(map[string]repository), - cache: cache.NewObjectLRU(maxCacheSize), - } -} - -// Add inserts a new repository in the pool. -func (p *RepositoryPool) Add(repo repository) error { - id := repo.ID() - if r, ok := p.repositories[id]; ok { - return errRepoAlreadyRegistered.New(r.Path()) - } - - p.idOrder = append(p.idOrder, id) - p.repositories[id] = repo - - return nil -} - -// AddGit adds a git repository to the pool. It also sets its path as ID. -func (p *RepositoryPool) AddGit(path string) error { - return p.AddGitWithID(path, path) -} - -// AddGitWithID adds a git repository to the pool. ID should be specified. -func (p *RepositoryPool) AddGitWithID(id, path string) error { - return p.Add(gitRepo(id, path, p.cache)) -} - -// AddSivaFile adds a siva file to the pool. It also sets its path as ID. -func (p *RepositoryPool) AddSivaFile(path string) error { - return p.Add(sivaRepo(path, path, p.cache)) -} - -// AddSivaFileWithID adds a siva file to the pool. ID should be specified. -func (p *RepositoryPool) AddSivaFileWithID(id, path string) error { - return p.Add(sivaRepo(id, path, p.cache)) -} - -// GetPos retrieves a repository at a given position. If the position is -// out of bounds it returns io.EOF. -func (p *RepositoryPool) GetPos(pos int) (*Repository, error) { - if pos >= len(p.repositories) { - return nil, io.EOF + cache: cache.NewObjectLRU(maxCacheSize), + library: lib, } - - id := p.idOrder[pos] - if id == "" { - return nil, io.EOF - } - - return p.GetRepo(id) } // ErrPoolRepoNotFound is returned when a repository id is not present in the pool. @@ -263,38 +90,52 @@ var ErrPoolRepoNotFound = errors.NewKind("repository id %s not found in the pool // GetRepo returns a repository with the given id from the pool. func (p *RepositoryPool) GetRepo(id string) (*Repository, error) { - r, ok := p.repositories[id] - if !ok { - return nil, ErrPoolRepoNotFound.New(id) + i := borges.RepositoryID(id) + + repo, err := p.library.Get(i, borges.ReadOnlyMode) + if err != nil { + if borges.ErrRepositoryNotExists.Is(err) { + return nil, ErrPoolRepoNotFound.New(id) + } + + return nil, err } - return r.Repo() + r := NewRepository(p.library, repo, p.cache) + return r, nil } // RepoIter creates a new Repository iterator func (p *RepositoryPool) RepoIter() (*RepositoryIter, error) { + it, err := p.library.Repositories(borges.ReadOnlyMode) + if err != nil { + return nil, err + } + iter := &RepositoryIter{ pool: p, + iter: it, } - atomic.StoreInt32(&iter.pos, 0) return iter, nil } // RepositoryIter iterates over all repositories in the pool type RepositoryIter struct { - pos int32 pool *RepositoryPool + iter borges.RepositoryIterator } // Next retrieves the next Repository. It returns io.EOF as error // when there are no more Repositories to retrieve. func (i *RepositoryIter) Next() (*Repository, error) { - pos := int(atomic.LoadInt32(&i.pos)) - r, err := i.pool.GetPos(pos) - atomic.AddInt32(&i.pos, 1) + repo, err := i.iter.Next() + if err != nil { + return nil, err + } - return r, err + r := NewRepository(i.pool.library, repo, i.pool.cache) + return r, nil } // Close finished iterator. It's no-op. diff --git a/repository_pool_test.go b/repository_pool_test.go index 835778315..5d51e4b62 100644 --- a/repository_pool_test.go +++ b/repository_pool_test.go @@ -4,37 +4,28 @@ import ( "io" "os" "path/filepath" - "strconv" + "strings" "testing" + fixtures "github.com/src-d/go-git-fixtures" "github.com/stretchr/testify/require" - "github.com/src-d/go-git-fixtures" - "gopkg.in/src-d/go-git.v4" + git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" ) -func TestRepository(t *testing.T) { +func TestRepositoryPoolBasic(t *testing.T) { require := require.New(t) - gitRepo := &git.Repository{} - repo := NewRepository("identifier", gitRepo, nil) - - require.Equal("identifier", repo.ID) - require.Equal(gitRepo, repo.Repository) - - repo = NewRepository("/other/path", nil, nil) - - require.Equal("/other/path", repo.ID) - require.Nil(repo.Repository) -} + lib, err := newMultiLibrary() + require.NoError(err) -func TestRepositoryPoolBasic(t *testing.T) { - require := require.New(t) + pool := NewRepositoryPool(cache.DefaultMaxSize, lib) - pool := NewRepositoryPool(cache.DefaultMaxSize) + iter, err := pool.RepoIter() + require.NoError(err) - repo, err := pool.GetPos(0) + repo, err := iter.Next() require.Nil(repo) require.Equal(io.EOF, err) @@ -42,37 +33,28 @@ func TestRepositoryPoolBasic(t *testing.T) { require.Nil(repo) require.EqualError(err, ErrPoolRepoNotFound.New("foo").Error()) - pool.Add(gitRepo("0", "/directory/should/not/exist", pool.cache)) - repo, err = pool.GetPos(0) + repo, err = pool.GetRepo("directory/should/not/exist") require.Nil(repo) - require.EqualError(err, git.ErrRepositoryNotExists.Error()) - - _, err = pool.GetPos(1) - require.Equal(io.EOF, err) + require.EqualError(err, ErrPoolRepoNotFound.New("directory/should/not/exist").Error()) path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - err = pool.Add(gitRepo("1", path, pool.cache)) + err = lib.AddPlain("1", path, nil) require.NoError(err) - repo, err = pool.GetPos(1) + iter, err = pool.RepoIter() require.NoError(err) - require.Equal("1", repo.ID) + repo, err = iter.Next() + require.NoError(err) + require.Equal("1", repo.ID()) require.NotNil(repo) repo, err = pool.GetRepo("1") require.NoError(err) - require.Equal("1", repo.ID) + require.Equal("1", repo.ID()) require.NotNil(repo) - err = pool.Add(gitRepo("1", path, pool.cache)) - require.Error(err) - require.True(errRepoAlreadyRegistered.Is(err)) - - _, err = pool.GetPos(0) - require.Equal(git.ErrRepositoryNotExists, err) - - _, err = pool.GetPos(2) + _, err = iter.Next() require.Equal(io.EOF, err) } @@ -81,12 +63,18 @@ func TestRepositoryPoolGit(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - pool := NewRepositoryPool(cache.DefaultMaxSize) + lib, err := newMultiLibrary() + require.NoError(err) + + pool := NewRepositoryPool(cache.DefaultMaxSize, lib) - require.NoError(pool.AddGit(path)) + require.NoError(lib.AddPlain(path, path, nil)) - repo, err := pool.GetPos(0) - require.Equal(path, repo.ID) + riter, err := pool.RepoIter() + require.NoError(err) + repo, err := riter.Next() + name := strings.TrimLeft(path, string(os.PathSeparator)) + require.Equal(name, repo.ID()) require.NotNil(repo) require.NoError(err) @@ -116,15 +104,21 @@ func TestRepositoryPoolIterator(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - pool := NewRepositoryPool(cache.DefaultMaxSize) - pool.Add(gitRepo("0", path, pool.cache)) - pool.Add(gitRepo("1", path, pool.cache)) + lib, err := newMultiLibrary() + require.NoError(err) + + pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + err = lib.AddPlain("0", path, nil) + require.NoError(err) + err = lib.AddPlain("1", path, nil) + require.NoError(err) iter, err := pool.RepoIter() require.NoError(err) count := 0 + var ids []string for { repo, err := iter.Next() if err != nil { @@ -133,25 +127,26 @@ func TestRepositoryPoolIterator(t *testing.T) { } require.NotNil(repo) - require.Equal(strconv.Itoa(count), repo.ID) + ids = append(ids, repo.ID()) count++ } require.Equal(2, count) + require.ElementsMatch([]string{"0", "1"}, ids) } func TestRepositoryPoolSiva(t *testing.T) { require := require.New(t) - expectedRepos := 3 + lib, err := newMultiLibrary() + require.NoError(err) - pool := NewRepositoryPool(cache.DefaultMaxSize) - path := filepath.Join( - os.Getenv("GOPATH"), - "src", "github.com", "src-d", "gitbase", - "_testdata", - ) + cwd, err := os.Getwd() + require.NoError(err) + + pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + path := filepath.Join(cwd, "_testdata") require.NoError( filepath.Walk(path, func(path string, info os.FileInfo, err error) error { @@ -160,20 +155,32 @@ func TestRepositoryPoolSiva(t *testing.T) { } if IsSivaFile(path) { - require.NoError(pool.AddSivaFile(path)) + require.NoError(lib.AddSiva(path, nil)) } return nil }), ) - require.Equal(expectedRepos, len(pool.repositories)) + expectedRepos := 5 + expected := map[string]int{ + "015da2f4-6d89-7ec8-5ac9-a38329ea875b": 606, + "015dcc49-9049-b00c-ba72-b6f5fa98cbe7": 68, + "015dcc49-90e6-34f2-ac03-df879ee269f3": 21, + "015dcc4d-0bdf-6aff-4aac-ffe68c752eb3": 380, + "015dcc4d-2622-bdac-12a5-ec441e3f3508": 72, + } + result := make(map[string]int) - expected := []int{606, 380, 69} - result := make([]int, expectedRepos) + it, err := pool.RepoIter() + require.NoError(err) - for i := 0; i < expectedRepos; i++ { - repo, err := pool.GetPos(i) + var i int + for { + repo, err := it.Next() + if err == io.EOF { + break + } require.NoError(err) iter, err := repo.Log(&git.LogOptions{ @@ -181,11 +188,16 @@ func TestRepositoryPoolSiva(t *testing.T) { }) require.NoError(err) + id := repo.ID() + result[id] = 0 require.NoError(iter.ForEach(func(c *object.Commit) error { - result[i]++ + result[id]++ return nil })) + + i++ } + require.Equal(expectedRepos, i) require.Equal(expected, result) } diff --git a/squash_iterator.go b/squash_iterator.go index b52a272cc..b45b3ae5f 100644 --- a/squash_iterator.go +++ b/squash_iterator.go @@ -92,7 +92,7 @@ func (i *squashReposIter) Advance() error { i.done = true - i.row = sql.NewRow(i.repo.ID) + i.row = sql.NewRow(i.repo.ID()) if i.filters != nil { ok, err := evalFilters(i.ctx, i.row, i.filters) if err != nil { @@ -202,7 +202,7 @@ func (i *squashRemoteIter) Advance() error { } i.remote = &Remote{ - RepoID: i.repo.ID, + RepoID: i.repo.ID(), Name: config.Name, URL: config.URLs[i.urlPos], Fetch: remoteFetchURL(config, i.urlPos), @@ -291,7 +291,7 @@ func (i *squashRepoRemotesIter) Advance() error { if err != nil { logrus.WithFields(logrus.Fields{ "iter": "repoRemoteIter", - "repo": i.repos.Repository().ID, + "repo": i.repos.Repository().ID(), "error": err, }).Error("unable to retrieve repository remotes") @@ -326,7 +326,7 @@ func (i *squashRepoRemotesIter) Advance() error { } i.remote = &Remote{ - RepoID: i.repos.Repository().ID, + RepoID: i.repos.Repository().ID(), Name: config.Name, URL: config.URLs[i.urlPos], Fetch: remoteFetchURL(config, i.urlPos), @@ -483,8 +483,8 @@ func (i *squashRefIter) Advance() error { continue } - i.ref = &Ref{i.repo.ID, ref} - i.row = referenceToRow(i.repo.ID, ref) + i.ref = &Ref{i.repo.ID(), ref} + i.row = referenceToRow(i.repo.ID(), ref) if i.filters != nil { ok, err := evalFilters(i.ctx, i.row, i.filters) @@ -540,7 +540,7 @@ func (i *squashRefIndexIter) New(ctx *sql.Context, repo *Repository) (ChainableI return nil, err } - values, err := i.index.Values(RepositoryPartition(repo.ID)) + values, err := i.index.Values(RepositoryPartition(repo.ID())) if err != nil { return nil, err } @@ -573,7 +573,7 @@ func (i *squashRefIndexIter) Advance() error { } repoID := i.row[0].(string) - if i.repo == nil || repoID != i.repo.ID { + if i.repo == nil || repoID != i.repo.ID() { i.repo, err = i.pool.GetRepo(repoID) if err != nil { if i.skipGitErrors { @@ -686,7 +686,7 @@ func (i *squashRepoRefsIter) Advance() error { if err != nil { logrus.WithFields(logrus.Fields{ "error": err, - "repo": i.repos.Repository().ID, + "repo": i.repos.Repository().ID(), }).Error("unable to retrieve references") if i.skipGitErrors { @@ -728,7 +728,7 @@ func (i *squashRepoRefsIter) Advance() error { continue } - i.ref = &Ref{i.repos.Repository().ID, ref} + i.ref = &Ref{i.repos.Repository().ID(), ref} if i.virtual { i.row = i.repos.Row() } else { @@ -824,7 +824,7 @@ func (i *squashRemoteRefsIter) Advance() error { if err != nil { logrus.WithFields(logrus.Fields{ "error": err, - "repo": i.Repository().ID, + "repo": i.Repository().ID(), }).Error("unable to retrieve references") if i.skipGitErrors { @@ -969,7 +969,7 @@ func (i *squashRefRefCommitsIter) Advance() error { commit, err := resolveCommit(i.Repository(), i.refs.Ref().Hash()) if err != nil { logrus.WithFields(logrus.Fields{ - "repo": i.Repository().ID, + "repo": i.Repository().ID(), "error": err, }).Error("unable to get commit") @@ -996,7 +996,7 @@ func (i *squashRefRefCommitsIter) Advance() error { i.commit = commit i.row = append( i.refs.Row(), - i.Repository().ID, + i.Repository().ID(), commit.Hash.String(), i.refs.Ref().Name().String(), int64(idx), @@ -1075,7 +1075,7 @@ func (i *squashRefHeadRefCommitsIter) Advance() error { i.commit, err = resolveCommit(i.Repository(), i.refs.Ref().Hash()) if err != nil { logrus.WithFields(logrus.Fields{ - "repo": i.Repository().ID, + "repo": i.Repository().ID(), "error": err, }).Error("unable to get commit") @@ -1088,7 +1088,7 @@ func (i *squashRefHeadRefCommitsIter) Advance() error { i.row = append( i.refs.Row(), - i.Repository().ID, + i.Repository().ID(), i.commit.Hash.String(), i.refs.Ref().Name().String(), int64(0), @@ -1142,7 +1142,7 @@ func (i *squashRefCommitsIndexIter) New(ctx *sql.Context, repo *Repository) (Cha return nil, err } - values, err := i.index.Values(RepositoryPartition(repo.ID)) + values, err := i.index.Values(RepositoryPartition(repo.ID())) if err != nil { return nil, err } @@ -1165,7 +1165,7 @@ func (i *squashRefCommitsIndexIter) Advance() error { } repoID := i.row[0] - if i.repo == nil || repoID != i.repo.ID { + if i.repo == nil || repoID != i.repo.ID() { i.repo.Close() i.repo, err = i.pool.GetRepo(i.row[0].(string)) @@ -1259,7 +1259,7 @@ func (i *squashRefCommitCommitsIter) Advance() error { commit := i.refCommits.Commit() i.row = append( i.refCommits.Row(), - commitToRow(i.Repository().ID, commit)..., + commitToRow(i.Repository().ID(), commit)..., ) if i.filters != nil { @@ -1321,7 +1321,7 @@ func (i *squashCommitsIter) New(ctx *sql.Context, repo *Repository) (ChainableIt }) if err != nil { logrus.WithFields(logrus.Fields{ - "repo": repo.ID, + "repo": repo.ID(), "error": err, }).Error("unable to get commit iterator") @@ -1365,7 +1365,7 @@ func (i *squashCommitsIter) Advance() error { return err } - i.row = commitToRow(i.repo.ID, i.commit) + i.row = commitToRow(i.repo.ID(), i.commit) if i.filters != nil { ok, err := evalFilters(i.ctx, i.row, i.filters) @@ -1418,7 +1418,7 @@ func (i *squashCommitsIndexIter) New(ctx *sql.Context, repo *Repository) (Chaina return nil, err } - values, err := i.index.Values(RepositoryPartition(repo.ID)) + values, err := i.index.Values(RepositoryPartition(repo.ID())) if err != nil { return nil, err } @@ -1445,7 +1445,7 @@ func (i *squashCommitsIndexIter) Advance() error { return err } - if i.repo == nil || i.repo.ID != i.iter.repoID { + if i.repo == nil || i.repo.ID() != i.iter.repoID { i.repo.Close() i.repo, err = i.pool.GetRepo(i.iter.repoID) @@ -1551,7 +1551,7 @@ func (i *squashRepoCommitsIter) Advance() error { }) if err != nil { logrus.WithFields(logrus.Fields{ - "repo": i.repos.Repository().ID, + "repo": i.repos.Repository().ID(), "error": err, }).Error("unable to get commit iterator") @@ -1576,7 +1576,7 @@ func (i *squashRepoCommitsIter) Advance() error { i.row = append( i.repos.Row(), - commitToRow(i.repos.Repository().ID, i.commit)..., + commitToRow(i.repos.Repository().ID(), i.commit)..., ) if i.filters != nil { @@ -1681,7 +1681,7 @@ func (i *squashRefHeadCommitsIter) Advance() error { } else { i.row = append( i.refs.Row(), - commitToRow(i.Repository().ID, i.commit)..., + commitToRow(i.Repository().ID(), i.commit)..., ) } @@ -1749,7 +1749,7 @@ func (i *squashCommitTreesIndexIter) New(ctx *sql.Context, repo *Repository) (Ch return nil, err } - values, err := i.index.Values(RepositoryPartition(repo.ID)) + values, err := i.index.Values(RepositoryPartition(repo.ID())) if err != nil { return nil, err } @@ -1772,7 +1772,7 @@ func (i *squashCommitTreesIndexIter) Advance() error { } repoID := i.row[0] - if i.repo == nil || repoID != i.repo.ID { + if i.repo == nil || repoID != i.repo.ID() { i.repo, err = i.pool.GetRepo(i.row[0].(string)) if err != nil { if i.skipGitErrors { @@ -1920,7 +1920,7 @@ func (i *squashCommitTreesIter) Advance() error { } else { i.row = append( i.commits.Row(), - i.Repository().ID, + i.Repository().ID(), i.commits.Commit().Hash.String(), i.tree.Hash.String(), ) @@ -2038,7 +2038,7 @@ func (i *squashRepoTreeEntriesIter) Advance() error { i.cursor++ i.row = append( i.repos.Row(), - treeEntryToRow(i.Repository().ID, i.entry)..., + treeEntryToRow(i.Repository().ID(), i.entry)..., ) if i.filters != nil { @@ -2141,7 +2141,7 @@ func (i *squashCommitMainTreeIter) Advance() error { } else { i.row = append( i.commits.Row(), - i.Repository().ID, + i.Repository().ID(), i.commits.Commit().Hash.String(), i.tree.Hash.String(), ) @@ -2358,7 +2358,7 @@ func (i *squashTreeEntriesIter) Advance() error { i.entry = &TreeEntry{i.tree.Hash, i.tree.Entries[i.cursor]} i.cursor++ - i.row = treeEntryToRow(i.repo.ID, i.entry) + i.row = treeEntryToRow(i.repo.ID(), i.entry) if i.filters != nil { ok, err := evalFilters(i.ctx, i.row, i.filters) @@ -2404,7 +2404,7 @@ func (i *squashTreeEntriesIndexIter) New(ctx *sql.Context, repo *Repository) (Ch return nil, err } - values, err := i.index.Values(RepositoryPartition(repo.ID)) + values, err := i.index.Values(RepositoryPartition(repo.ID())) if err != nil { return nil, err } @@ -2432,7 +2432,7 @@ func (i *squashTreeEntriesIndexIter) Advance() error { return err } - if i.repo == nil || i.repo.ID != i.iter.repoID { + if i.repo == nil || i.repo.ID() != i.iter.repoID { i.repo, err = i.pool.GetRepo(i.iter.repoID) if err != nil { if i.skipGitErrors { @@ -2544,7 +2544,7 @@ func (i *squashTreeTreeEntriesIter) Advance() error { } else { i.row = append( i.trees.Row(), - treeEntryToRow(i.Repository().ID, i.entry)..., + treeEntryToRow(i.Repository().ID(), i.entry)..., ) } @@ -2605,7 +2605,7 @@ func (i *squashCommitBlobsIndexIter) New(ctx *sql.Context, repo *Repository) (Ch return nil, err } - values, err := i.index.Values(RepositoryPartition(repo.ID)) + values, err := i.index.Values(RepositoryPartition(repo.ID())) if err != nil { return nil, err } @@ -2628,7 +2628,7 @@ func (i *squashCommitBlobsIndexIter) Advance() error { } repoID := i.row[0] - if i.repo == nil || repoID != i.repo.ID { + if i.repo == nil || repoID != i.repo.ID() { i.repo, err = i.pool.GetRepo(i.row[0].(string)) if err != nil { if i.skipGitErrors { @@ -2750,7 +2750,7 @@ func (i *squashCommitBlobsIter) Advance() error { i.tree, err = i.Repository().TreeObject(i.commits.Commit().TreeHash) if err != nil { logrus.WithFields(logrus.Fields{ - "repo": i.Repository().ID, + "repo": i.Repository().ID(), "tree_hash": i.commits.Commit().TreeHash.String(), "error": err, }).Error("unable to retrieve tree object") @@ -2782,7 +2782,7 @@ func (i *squashCommitBlobsIter) Advance() error { i.row = append( i.commits.Row(), - i.Repository().ID, + i.Repository().ID(), i.commits.Commit().Hash.String(), i.file.Blob.Hash.String(), ) @@ -2884,7 +2884,7 @@ func (i *squashRepoBlobsIter) Advance() error { return err } - row, err := blobToRow(i.Repository().ID, i.blob, i.readContent) + row, err := blobToRow(i.Repository().ID(), i.blob, i.readContent) if err != nil { return err } @@ -2978,7 +2978,7 @@ func (i *squashTreeEntryBlobsIter) Advance() error { i.blob, err = i.Repository().BlobObject(entry.Hash) if err != nil { logrus.WithFields(logrus.Fields{ - "repo": i.Repository().ID, + "repo": i.Repository().ID(), "error": err, "blob": entry.Hash, }).Error("blob object found not be found") @@ -2990,7 +2990,7 @@ func (i *squashTreeEntryBlobsIter) Advance() error { return err } - row, err := blobToRow(i.Repository().ID, i.blob, i.readContent) + row, err := blobToRow(i.Repository().ID(), i.blob, i.readContent) if err != nil { return err } @@ -3069,7 +3069,7 @@ func (i *squashCommitBlobBlobsIter) Advance() error { } blob := i.commitBlobs.Blob() - row, err := blobToRow(i.Repository().ID, blob, i.readContent) + row, err := blobToRow(i.Repository().ID(), blob, i.readContent) if err != nil { return err } @@ -3162,7 +3162,7 @@ func (i *squashCommitFilesIter) Advance() error { if i.skipGitErrors { logrus.WithFields(logrus.Fields{ "err": err, - "repo": i.Repository().ID, + "repo": i.Repository().ID(), "commit": i.commit.Hash.String(), }).Error("could not get files for commit") continue @@ -3183,7 +3183,7 @@ func (i *squashCommitFilesIter) Advance() error { if i.skipGitErrors { logrus.WithFields(logrus.Fields{ "err": err, - "repo": i.Repository().ID, + "repo": i.Repository().ID(), "commit": i.commit.Hash.String(), }).Error("could not get files for commit") continue @@ -3248,7 +3248,7 @@ func NewIndexCommitFilesIter(index sql.IndexLookup, filters sql.Expression) File } func (i *squashIndexCommitFilesIter) New(ctx *sql.Context, repo *Repository) (ChainableIter, error) { - values, err := i.index.Values(RepositoryPartition(repo.ID)) + values, err := i.index.Values(RepositoryPartition(repo.ID())) if err != nil { return nil, err } @@ -3281,7 +3281,7 @@ func (i *squashIndexCommitFilesIter) Advance() error { i.file = commitFile.File - if i.repo == nil || i.repo.ID != commitFile.Repository { + if i.repo == nil || i.repo.ID() != commitFile.Repository { i.repo.Close() i.repo, err = i.pool.GetRepo(commitFile.Repository) @@ -3379,7 +3379,7 @@ func (i *squashCommitFileFilesIter) Advance() error { } f := i.files.File() - row, err := fileToRow(i.Repository().ID, i.files.TreeHash(), f, i.readContent) + row, err := fileToRow(i.Repository().ID(), i.files.TreeHash(), f, i.readContent) if err != nil { return err } @@ -3454,7 +3454,7 @@ func (i *squashCommitFileBlobsIter) Advance() error { } f := i.files.File() - row, err := blobToRow(i.Repository().ID, &f.Blob, i.readContent) + row, err := blobToRow(i.Repository().ID(), &f.Blob, i.readContent) if err != nil { return err } diff --git a/squash_iterator_test.go b/squash_iterator_test.go index 744631614..b7462d0ce 100644 --- a/squash_iterator_test.go +++ b/squash_iterator_test.go @@ -2,14 +2,17 @@ package gitbase import ( "context" + "os" "path/filepath" "testing" + "github.com/src-d/go-borges" + "github.com/src-d/go-borges/plain" fixtures "github.com/src-d/go-git-fixtures" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-billy.v4/osfs" ) func TestAllReposIter(t *testing.T) { @@ -40,10 +43,14 @@ func TestSquashContextCancelled(t *testing.T) { session, err := getSession(ctx) require.NoError(err) - for _, it := range iters { - repo, err := session.Pool.GetPos(0) - require.NoError(err) + pool := session.Pool + iter, err := pool.RepoIter() + require.NoError(err) + repo, err := iter.Next() + require.NoError(err) + + for _, it := range iters { iter, err := it.New(ctx, repo) require.NoError(err) @@ -739,12 +746,14 @@ func TestCommitFileBlobsIter(t *testing.T) { } func chainableIterRowsError(t *testing.T, ctx *sql.Context, iter ChainableIter) { + t.Helper() table := newSquashTable(iter) _, err := tableToRows(ctx, table) require.Error(t, err) } func chainableIterRows(t *testing.T, ctx *sql.Context, iter ChainableIter) []sql.Row { + t.Helper() table := newSquashTable(iter) rows, err := tableToRows(ctx, table) require.NoError(t, err) @@ -752,29 +761,39 @@ func chainableIterRows(t *testing.T, ctx *sql.Context, iter ChainableIter) []sql } func setupIter(t *testing.T) (*sql.Context, func()) { + t.Helper() return setupIterWithErrors(t, false, false) } func setupIterWithErrors(t *testing.T, badRepo bool, skipErrors bool) (*sql.Context, func()) { - pool := NewRepositoryPool(cache.DefaultMaxSize) + t.Helper() + require := require.New(t) + + lib, pool, err := newMultiPool() + require.NoError(err) + if badRepo { // TODO: add repo with errors - pool.Add(gitRepo("bad_repo", "bad_path", pool.cache)) + loc, err := plain.NewLocation(borges.LocationID("bad"), osfs.New("/does/not/exist"), nil) + require.NoError(err) + + lib.plain.AddLocation(loc) } for _, f := range fixtures.ByTag("worktree") { path := f.Worktree().Root() ok, err := IsGitRepo(path) - require.NoError(t, err) + require.NoError(err) if ok { - pool.AddGit(f.Worktree().Root()) + err = lib.AddPlain(pathToName(path), path, nil) + require.NoError(err) } } session := NewSession(pool, WithSkipGitErrors(skipErrors)) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) cleanup := func() { - require.NoError(t, fixtures.Clean()) + require.NoError(fixtures.Clean()) } return ctx, cleanup @@ -948,9 +967,16 @@ func setupWithIndex( } func TestRefsIterSiva(t *testing.T) { - path := filepath.Join("_testdata", "05893125684f2d3943cd84a7ab2b75e53668fba1.siva") - pool := NewRepositoryPool(cache.DefaultMaxSize) - require.NoError(t, pool.AddSivaFile(path)) + cwd, err := os.Getwd() + require.NoError(t, err) + + path := filepath.Join(cwd, "_testdata", "05893125684f2d3943cd84a7ab2b75e53668fba1.siva") + + lib, pool, err := newMultiPool() + require.NoError(t, err) + + err = lib.AddSiva(path, nil) + require.NoError(t, err) session := NewSession(pool) ctx := sql.NewContext(context.Background(), sql.WithSession(session)) @@ -965,13 +991,13 @@ func TestRefsIterSiva(t *testing.T) { expected := []sql.Row{ { - path, - "refs/heads/HEAD/015da2f4-6d89-7ec8-5ac9-a38329ea875b", + "015da2f4-6d89-7ec8-5ac9-a38329ea875b", + "HEAD", "dbfab055c70379219cbcf422f05316fdf4e1aed3", }, { - path, - "refs/heads/master/015da2f4-6d89-7ec8-5ac9-a38329ea875b", + "015da2f4-6d89-7ec8-5ac9-a38329ea875b", + "refs/heads/master", "dbfab055c70379219cbcf422f05316fdf4e1aed3", }, } diff --git a/tree_entries.go b/tree_entries.go index 5724ccb5c..4c7a1caca 100644 --- a/tree_entries.go +++ b/tree_entries.go @@ -199,7 +199,7 @@ func (i *treeEntriesRowIter) next() (sql.Row, error) { entry := &TreeEntry{i.tree.Hash, i.tree.Entries[i.cursor]} i.cursor++ - return treeEntryToRow(i.repo.ID, entry), nil + return treeEntryToRow(i.repo.ID(), entry), nil } } @@ -231,7 +231,7 @@ func (i *treeEntriesRowIter) nextByHash() (sql.Row, error) { entry := &TreeEntry{i.tree.Hash, i.tree.Entries[i.cursor]} i.cursor++ - return treeEntryToRow(i.repo.ID, entry), nil + return treeEntryToRow(i.repo.ID(), entry), nil } } @@ -343,8 +343,7 @@ func newTreeEntriesKeyValueIter( return nil, err } - r := pool.repositories[repo.ID] - idx, err := newRepositoryIndex(r) + idx, err := newRepositoryIndex(repo) if err != nil { return nil, err } @@ -388,7 +387,7 @@ func (i *treeEntriesKeyValueIter) Next() ([]interface{}, []byte, error) { } key, err := encodeIndexKey(&treeEntriesIndexKey{ - Repository: i.repo.ID, + Repository: i.repo.ID(), Packfile: packfile.String(), Offset: offset, Pos: i.pos - 1, @@ -398,7 +397,7 @@ func (i *treeEntriesKeyValueIter) Next() ([]interface{}, []byte, error) { return nil, nil, err } - row := treeEntryToRow(i.repo.ID, &TreeEntry{i.tree.Hash, entry}) + row := treeEntryToRow(i.repo.ID(), &TreeEntry{i.tree.Hash, entry}) values, err := rowIndexValues(row, i.columns, TreeEntriesSchema) if err != nil { return nil, nil, err diff --git a/tree_entries_test.go b/tree_entries_test.go index 18bc8241d..75e8cb7f4 100644 --- a/tree_entries_test.go +++ b/tree_entries_test.go @@ -3,10 +3,10 @@ package gitbase import ( "testing" - "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-git.v4/plumbing" "github.com/src-d/go-mysql-server/sql" "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing" ) func TestTreeEntriesTable(t *testing.T) {