Allow global configuration of gcloud

[ryanseys]

From discussion at #168 (comment)

Support supplying credentials earlier in the gcloud setup process.

var gcloud = require('gcloud');
gcloud.options({ keyFilename: !ON_COMPUTE && '/path/to/the/key.json' });

[stephenplusplus]

#46

If we're continuing the discussion here, I'll just add a couple notes.

// Would be nice if it worked on gce
var gcloud = require('gcloud');
var dataset = new gcloud.datastore.Dataset({ projectId: 'hi' });

// Should work on gae & elsewhere
var gcloud = require('gcloud')({ credentials: {/*... */} });
var dataset = new gcloud.datastore.Dataset({ projectId: 'hi' });

So, a couple things are different from your example:

1. No need for a user on gce to require any connection configuration. That can be handled for them by us like it is currently (right?)
2. Remove the .options method, and instead pass the credentials/keyfilename to the gcloud required object.

Here's how I would envision that working...

// lib/index.js
var connection = require('./common/connection.js');
function gcloud(connectionOptions) {
  var conn = new connection(connectionOptions);
  gcloud.datastore.connection = conn;
  gcloud.pubsub.connection = conn;
  gcloud.storage.connection = conn;
}
gcloud.datastore = require('./datastore/index.js');
gcloud.pubsub = require('./pubsub/index.js');
gcloud.storage = require('./storage/index.js');

module.exports = gcloud;

// lib/storage/index.js
var connection = require('./common/connection.js');
var conn = module.exports.connection;
if (!module.exports.connection) {
  // no connection was configured, so we're on gce
  conn = new connection();
}
function Bucket() {}
// ...
module.exports.Bucket = Bucket;

If that example is awful, feel free to re-think it completely. Trying to wrangle a baby over here, so my thought 🚋 could be way off :)

[jgeewax]

I'd like to take this to an extra level and include projectId in the connection parameters (it's uncommon for a single chunk of code to operate on more than one project simultaneously).

That is:

var gcloud = require('gcloud')({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});

// This is a weird one because there's currently one
// dataset per project, but this may change over time.
var dataset = new gcloud.datastore.Dataset() 

// This one is pretty standard...
var bucket = new gcloud.storage.Bucket({ bucketName: 'my-bucket' });

If you do want to do more than one project, the old config should still be able to be overridden:

// Assuming we have gcloud set as above...
var otherDataset = new gcloud.datastore.Dataset({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});

Or even:

var gcloud = require('gcloud');

var projectOne = gcloud.configure({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});

var projectTwo = gcloud.configure({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});

var bucketOne = new projectOne.storage.Bucket( ... );
var bucketTwo = new projectTwo.storage.Bucket( ... );

Thoughts?

CC: @proppy @ryanseys @stephenplusplus @silvolu

[jgeewax]

(I should add that if we allow it to be done this way, the Authentication section of the README becomes much easier to write, as the demo isn't service-specific... See #193 )

[stephenplusplus]

I'd like to take this to an extra level and include projectId in the connection parameters (it's uncommon for a single chunk of code to operate on more than one project simultaneously).

Makes sense 👍

How about still avoiding a new method:

var gcloud = require('gcloud');

// one project.
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});

// another project.
var anotherProject = gcloud({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});

[jgeewax]

SGTM. Who wants to take this one?

[stephenplusplus]

I'm happy to give it a go, unless @ryanseys wants the honors 👑

[silvolu]

SGTM as long as we allow overwriting for a single service instance, e.g. as @jgeewax mentioned

var otherDataset = new gcloud.datastore.Dataset({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});

[proppy]

Note that on GCE you can query the projectId from the metadata server, and since you don't need the .json credentials either: var myProject = gcloud(); should just work.

[rakyll]

var dataset = new gcloud.datastore.Dataset()

Your service account may have read/right permissions for other projects. So, this is not a weird one.

[rakyll]

I should add that if we allow it to be done this way, the Authentication section of the README becomes much easier to write, as the demo isn't service-specific..

There is no service specific auth. All services work with a credentials object that looks more or less the same -- we can make the object exactly the same for each service.

The credentials object is the global configuration. We don't need need interfaces.

var conf = {
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
};

var ds = gcloud.datastore.dataset(conf);
// or on GCE
var ds = gcloud.datastore.dataset();
var bucket = gcloud.storage.bucket(conf);

This proposal is not looking good to me.

[stephenplusplus]

That covered the credentials, but not any further config options:

var conf = {
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
};
// ...
var bucket = gcloud.storage.bucket(conf); // bucketName?

It's not an exhaustive amount of effort we would force on the developer to figure that problem out-- they would probably just inline the entire configuration object again, as opposed to extending the object. However, if we know the developer is going to do something the same way each time they invoke a sub-module, why not plan for this and allow up-front specification?

var gcloud = require('gcloud')({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});
var ds = gcloud.datastore.dataset();
var bucket = gcloud.storage.bucket({ bucketName: 'hi' });

And from config-aware envs:

var gcloud = require('gcloud');
var ds = gcloud.datastore.dataset();
var bucket = gcloud.storage.bucket({ bucketName: 'hi' });

And if you need multiple connection details:

var gcloud = require('gcloud');
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});
var myOtherProject = gcloud({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});
var ds = myProject.datastore.dataset();
var bucket = myOtherProject.storage.bucket({ bucketName: 'hi' });

[stephenplusplus]

I would say, however, providing anything but connection info to the gcloud module can introduce some confusion. I would vote to keep it limited to just credentials or keyFilename. But, maybe there are use-cases where being allowed to specify additional defaults would be beneficial?

[rakyll]

But, maybe there are use-cases where being allowed to specify additional defaults would be beneficial?

There could be global options. But if there are not default, the configuration object on the service layer (called conf above) should require them as well.

Non-optional options can be set optionally via gcloud.configure(opts), so we still can eliminate gcloud initialisation duality.

var bucket = gcloud.storage.bucket(conf); // bucketName?

We actually never need a projectId field for datastore and storage clients. The configuration is already fragmented. The only option you can pass is the keyFilename or credentials.

Overriding creates more confusion.

// Confusion #1: Should this authorise via metadata server or not?
var gcloud = require('gcloud');
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset();


// Confusion #2: Will this override keyFilename or not?
var gcloud = require('gcloud');
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset({ credentials: blah });

// Confusion #3: Will this discover my GCE project Id or not?
var gcloud = require('gcloud');
var myProject = gcloud({
  keyFilename: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset();

// Confusion #4: Will this work on GAE devappserver?
var gcloud = require('gcloud');
var ds = myProject.datastore.dataset(); // this will work
var bucket = myProject.storage.bucket(); // this won't work
// Whereas on prod, both will work. So, if this is a GAE user, global config is not usable.

So, configuration object differs per service. Auth behaves differently according to the environment. Some services support devappserver, some don't.

Why do we need to mess with all the logic here (and explaining it) instead of asking user to repeat himself twice by passing a credentials object or a keyFilename string?

Our current way of doing auth is already extremely implicit -- GAE and GCE behaviours are magical. I don't want to add more confusion.