List endpoints

cmd/ghsync/main.go

@@ -17,7 +17,8 @@ var (
 var app = cli.New("ghsync", version, build, "GitHub metadata sync")
 
 func main() {
-	app.AddCommand(&subcmd.SyncCommand{})
+	app.AddCommand(&subcmd.ShallowCommand{})
+	app.AddCommand(&subcmd.DeepCommand{})
 	app.AddCommand(&subcmd.MigrateCommand{})
 
 	app.RunMain()

cmd/ghsync/subcmd/common.go

@@ -1,13 +1,24 @@
 package subcmd
 
 import (
+	"context"
+	"database/sql"
 	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
 
 	"github.com/src-d/ghsync/models/migrations"
+	"github.com/src-d/ghsync/utils"
+	"gopkg.in/src-d/go-log.v1"
 
 	"github.com/golang-migrate/migrate/v4"
 	_ "github.com/golang-migrate/migrate/v4/database/postgres"
 	bindata "github.com/golang-migrate/migrate/v4/source/go_bindata"
+	"github.com/google/go-github/github"
+	"github.com/gregjones/httpcache"
+	"github.com/gregjones/httpcache/diskcache"
+	"golang.org/x/oauth2"
 )
 
 const maxVersion uint = 1560510971
@@ -25,6 +36,45 @@ func (o PostgresOpt) URL() string {
 		o.User, o.Password, o.Host, o.Port, o.DB)
 }
 
+func (o PostgresOpt) initDB() (db *sql.DB, err error) {
+	db, err = sql.Open("postgres", o.URL())
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			db.Close()
+			db = nil
+		}
+	}()
+
+	if err = db.Ping(); err != nil {
+		return db, err
+	}
+
+	m, err := newMigrate(o.URL())
+	if err != nil {
+		return db, err
+	}
+
+	dbVersion, _, err := m.Version()
+
+	if err != nil && err != migrate.ErrNilVersion {
+		return db, err
+	}
+
+	if dbVersion != maxVersion {
+		return db, fmt.Errorf(
+			"database version mismatch. Current version is %v, but this binary needs version %v. "+
+				"Use the 'migrate' subcommand to upgrade your database", dbVersion, maxVersion)
+	}
+
+	log.With(log.Fields{"db-version": dbVersion}).Debugf("the DB version is up to date")
+	log.Infof("connection with the DB established")
+	return db, nil
+}
+
 func newMigrate(url string) (*migrate.Migrate, error) {
 	// wrap assets into Resource
 	s := bindata.Resource(migrations.AssetNames(),
@@ -38,3 +88,32 @@ func newMigrate(url string) (*migrate.Migrate, error) {
 	}
 	return migrate.NewWithSourceInstance("go-bindata", d, url)
 }
+
+func newClient(token string) (*github.Client, error) {
+	http := oauth2.NewClient(context.TODO(), oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: token},
+	))
+
+	dirPath := filepath.Join(os.TempDir(), "ghsync")
+	err := os.MkdirAll(dirPath, os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("error while creating directory %s: %v", dirPath, err)
+	}
+
+	t := httpcache.NewTransport(diskcache.New(dirPath))
+	t.Transport = &RemoveHeaderTransport{utils.NewRateLimitTransport(http.Transport)}
+	http.Transport = t
+
+	return github.NewClient(http), nil
+}
+
+type RemoveHeaderTransport struct {
+	T http.RoundTripper
+}
+
+func (t *RemoveHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.Header.Del("X-Ratelimit-Limit")
+	req.Header.Del("X-Ratelimit-Remaining")
+	req.Header.Del("X-Ratelimit-Reset")
+	return t.T.RoundTrip(req)
+}

cmd/ghsync/subcmd/deep.go

@@ -0,0 +1,63 @@
+package subcmd
+
+import (
+	"github.com/src-d/ghsync/deep"
+
+	"gopkg.in/src-d/go-cli.v0"
+	"gopkg.in/src-d/go-log.v1"
+	"gopkg.in/src-d/go-queue.v1"
+	_ "gopkg.in/src-d/go-queue.v1/amqp"
+	_ "gopkg.in/src-d/go-queue.v1/memory"
+)
+
+type DeepCommand struct {
+	cli.Command `name:"deep" short-description:"Deep sync of GitHub data" long-description:"Deep sync of GitHub data"`
+
+	Token string `long:"token" env:"GHSYNC_TOKEN" description:"GitHub personal access token" required:"true"`
+	Org   string `long:"org" env:"GHSYNC_ORG" description:"Name of the GitHub organization" required:"true"`
+
+	QueueOpt struct {
+		Queue  string `long:"queue" env:"GHSYNC_QUEUE" description:"queue name. If it's not set the organization name will be used"`
+		Broker string `long:"broker" env:"GHSYNC_BROKER" default:"amqp://localhost:5672" description:"broker service URI"`
+	} `group:"go-queue connection options"`
+
+	Postgres PostgresOpt `group:"PostgreSQL connection options"`
+}
+
+func (c *DeepCommand) Execute(args []string) error {
+	db, err := c.Postgres.initDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	client, err := newClient(c.Token)
+	if err != nil {
+		return err
+	}
+
+	broker, err := queue.NewBroker(c.QueueOpt.Broker)
+	if err != nil {
+		return err
+	}
+
+	qName := c.QueueOpt.Queue
+	if qName == "" {
+		qName = c.Org
+	}
+	queue, err := broker.Queue(qName)
+	if err != nil {
+		return err
+	}
+
+	syncer := deep.NewSyncer(db, client, queue)
+
+	go func() {
+		err := syncer.DoOrganization(c.Org)
+		if err != nil {
+			log.Errorf(err, "syncer.DoOrganization finished with error")
+		}
+	}()
+
+	return syncer.Wait()
+}

cmd/ghsync/subcmd/shallow.go

@@ -0,0 +1,32 @@
+package subcmd
+
+import (
+	"github.com/src-d/ghsync/shallow"
+
+	"gopkg.in/src-d/go-cli.v0"
+)
+
+type ShallowCommand struct {
+	cli.Command `name:"shallow" short-description:"Shallow sync of GitHub data" long-description:"Shallow sync of GitHub data"`
+
+	Token string `long:"token" env:"GHSYNC_TOKEN" description:"GitHub personal access token" required:"true"`
+	Org   string `long:"org" env:"GHSYNC_ORG" description:"Name of the GitHub organization" required:"true"`
+
+	Postgres PostgresOpt `group:"PostgreSQL connection options"`
+}
+
+func (c *ShallowCommand) Execute(args []string) error {
+	db, err := c.Postgres.initDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	client, err := newClient(c.Token)
+	if err != nil {
+		return err
+	}
+
+	orgSyncer := shallow.NewOrganizationSyncer(db, client)
+	return orgSyncer.Sync(c.Org)
+}

common.go → deep/common.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"strings"

issue.go → deep/issue.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

issue_comment.go → deep/issue_comment.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

organization.go → deep/organization.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

pull_request.go → deep/pull_request.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

pull_request_comment.go → deep/pull_request_comment.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

pull_request_review.go → deep/pull_request_review.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

repository.go → deep/repository.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

syncer.go → deep/syncer.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"database/sql"