How to integrate Google Bigquery with c# console application

Question

If it is possible to integrate Google big query with C# console application?.

If yes how we can do, i searched over internet i could not find proper answer for that.

I want connection string format? I have created Client ID from Google Developer console how authentication has done? It is one time configuration or every time we need to login in google account to authenticate.

If there is any sample application to connect sample data it would be helpful.

Thanks, Selvakumar S

Answer 1

In your answer i could not able to add namespace "using Google.Apis.Authentication.OAuth2.DotNetOpenAuth;"

But i manage to retrieve results from BigQuery using below code, you need to update Project Name, Project Id and Query.

Download Client ID (I am using Installed Application - Other category ) generate JSON file and add into your Debug folder.

using Google.Apis.Auth.OAuth2;
using System.IO;
using System.Threading;
using Google.Apis.Bigquery.v2;
using Google.Apis.Bigquery.v2.Data;
using System.Data;
using Google.Apis.Services;
using System;

namespace GoogleBigQuery
{
    public class Class1
    {
        private static void Main()
        {
            UserCredential credential;
            using (var stream = new FileStream("client_secrets.json", FileMode.Open,
                                            FileAccess.Read))
            {
                credential = GoogleWebAuthorizationBroker.AuthorizeAsync(
                   GoogleClientSecrets.Load(stream).Secrets,
                   new[] { BigqueryService.Scope.Bigquery },
                   "user", CancellationToken.None).Result;
            }

            //  Create and initialize the Bigquery service. Use the Project Name value
            //  from the New Project window for the ApplicationName variable.

            BigqueryService Service = new BigqueryService(new BaseClientService.Initializer()
            {
                HttpClientInitializer = credential,
                ApplicationName = "PROJECT NAME"
            });

            string query = "YOUR QUERY";

            JobsResource j = Service.Jobs;
            QueryRequest qr = new QueryRequest();
            qr.Query = query;

            DataTable DT = new DataTable();
            int i = 0;

            QueryResponse response = j.Query(qr, "PROJECT ID").Execute();

            if (response != null)
            {
                int colCount = response.Schema.Fields.Count;

                foreach (var Column in response.Schema.Fields)
                {
                    DT.Columns.Add(Column.Name);
                }

                foreach (TableRow row in response.Rows)
                {
                    DataRow dr = DT.NewRow();

                    for (i = 0; i < colCount; i++)
                    {
                        dr[i] = row.F[i].V;
                    }

                    DT.Rows.Add(dr);
                }
            }
            else
            {
                Console.WriteLine("Response is null");
            }
        }
    }
}

Thanks.

Answer 2

Here's a working sample based on another question in StackOverflow:

using DotNetOpenAuth.OAuth2;
using Google.Apis.Authentication.OAuth2;
using Google.Apis.Authentication.OAuth2.DotNetOpenAuth;

using Google.Apis.Bigquery.v2;
using Google.Apis.Bigquery.v2.Data;

using Google.Apis.Util;
using System;
using System.Diagnostics;
using System.Collections.Generic;

namespace BigQueryConsole
{
    public class BigQueryConsole
    {
        // Put your client ID and secret here (from https://developers.google.com/console)
        // Use the installed app flow here.
        // Client ID looks like "9999999.apps.googleusercontent.com"
        static string clientId = "YOURCLIENTID";  
        static string clientSecret = "YOURSECRET";

        // Project ID is in the URL of your project on the APIs Console
        // Project ID looks like "999999";
        static string projectId = "YOURPROJECTID";

        // Query in SQL-like form
        static string query = "SELECT state, count(*) from [publicdata:samples.natality] GROUP BY state ORDER BY state ASC";

        public static void Main(string[] args)
        {
            // Register an authenticator.
            var provider = new NativeApplicationClient(GoogleAuthenticationServer.Description);

            provider.ClientIdentifier = clientId;
            provider.ClientSecret = clientSecret;

            // Initiate an OAuth 2.0 flow to get an access token

            var auth = new OAuth2Authenticator<NativeApplicationClient>(provider, GetAuthorization);

            // Create the service.
            var service = new BigqueryService(auth);
            JobsResource j = service.Jobs;
            QueryRequest qr = new QueryRequest();
            qr.Query = query;

            QueryResponse response = j.Query(qr, projectId).Fetch();
            foreach (TableRow row in response.Rows)
            {
                List<string> list = new List<string>();
                foreach (TableRow.FData field in row.F)
                {
                    list.Add(field.V);
                }
                Console.WriteLine(String.Join("\t", list));
            }
            Console.WriteLine("\nPress enter to exit");
            Console.ReadLine();
        }

        private static IAuthorizationState GetAuthorization(NativeApplicationClient arg)
        {
            // Get the auth URL:
            IAuthorizationState state = new AuthorizationState(new[] {  BigqueryService.Scopes.Bigquery.GetStringValue() });
            state.Callback = new Uri(NativeApplicationClient.OutOfBandCallbackUrl);
            Uri authUri = arg.RequestUserAuthorization(state);

            // Request authorization from the user (by opening a browser window):
            Process.Start(authUri.ToString());
            Console.Write("  Authorization Code: ");
            string authCode = Console.ReadLine();
            Console.WriteLine();

            // Retrieve the access token by using the authorization code:
            return arg.ProcessUserAuthorization(authCode, state);
        }
    }
}

Answer 3

You can do this using Google's BigQuery .Net client library.

Please refer their documentation : Google.Cloud.BigQuery.V2

Before proceed you have to go to APIs & Services, then search for "BigQuery API" and enable it.

Then you have to install their client library using Nuget Package manager.

Install-Package Google.Cloud.BigQuery.V2

Then you need to download Service Account key for the authentication. For that you have to create a service account, then create a new key for that service account as json. Then you have to download that json file.

Then you can connect with your BigQuery as below:

static void Main(string[] args)
    {
        string keyPath = "path_to_downloaded_json_file";

        var credential = GoogleCredential.FromFile(keyPath);

        BigQueryClient client = BigQueryClient.Create("your_project_name", credential);
        
        BigQueryTable table = client.GetDataset("your_dataset_name").GetTable("your_data_table_name");
        
        string sql = $"SELECT your_column AS foo, your_other_column AS bar FROM {table}";
        BigQueryParameter[] parameters = null;
        BigQueryResults results = client.ExecuteQuery(sql, parameters);

        foreach (BigQueryRow row in results)
        {
            Console.WriteLine($"{row["foo"]}: {row["bar"]}");
        }
    }