Go Nested Json Unmarshal
GoでNestされたjsonの扱いがわからなかったのでメモ。
jStr := `
{
"root": {
"c1": ["cv1"],
"c2": ["cv2"]
}
}
`
type Children struct {
c1 []string `json:"c1"`
c2 []string `json:"c2"`
}
type Root struct {
c Children `json:"root"`
}
var root Root
if err := json.Unmarshal([]byte(jStr), &root); err != nil {
log.Fatal(err)
}
log.Printf("%+v\n", root)
このパッケージも良さそう。 github.com
Go からBIgQueryを扱うサンプル
Goを使ってAWSからBIgQueryにアクセスする必要があるときの処理。
package bigquery
import (
"log"
"reflect"
"io/ioutil"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/bigquery/v2"
)
const projectId = "***"
const dataSetId = "***"
const tableName = "***"
// Type:
// STRING, BYTES, INTEGER, FLOAT, BOOLEAN, TIMESTAMP, DATE, TIME, DATETIME,
type TableModel struct {
Name string `column:"name" bq_type:"STRING" bq_mode:"NULLABLE"`
Address string `column:"address" bq_type:"STRING" bq_mode:"NULLABLE"`
}
func CreateTableSchema(tableModel interface{}) bigquery.TableSchema {
var schemas = &bigquery.TableSchema{}
fields := []*bigquery.TableFieldSchema{}
rt := reflect.TypeOf(tableModel)
for i := 0; i < rt.NumField(); i++ {
field := rt.Field(i)
fields = append(fields,
&bigquery.TableFieldSchema{
Mode: field.Tag.Get("bq_mode"),
Name: field.Tag.Get("column"),
Type: field.Tag.Get("bq_type"),
},
)
}
schemas.Fields = fields
return *schemas
}
func CreateInsertRow(tableModel interface{}) []*bigquery.TableDataInsertAllRequestRows {
rows := make([]*bigquery.TableDataInsertAllRequestRows, 0)
row := &bigquery.TableDataInsertAllRequestRows{
Json: make(map[string]bigquery.JsonValue, 0),
}
rt := reflect.TypeOf(tableModel)
rv := reflect.ValueOf(tableModel)
for i := 0; i < rt.NumField(); i++ {
value := rv.Field(i).Interface()
field := rt.Field(i)
row.Json[field.Tag.Get("column")] = value
rows = append(rows, row)
}
return rows
}
func BqSample() {
pemKeyBytes, err := ioutil.ReadFile(<your file path>)
if err != nil {
panic(err)
}
t, err := google.JWTConfigFromJSON(pemKeyBytes, "https://www.googleapis.com/auth/bigquery")
if err != nil {
panic(err)
}
/*
Make DataSet
*/
client := t.Client(oauth2.NoContext)
service, _ := bigquery.New(client)
datasetList, _ := service.Datasets.List(projectId).All(false).Do()
for _, dataset := range datasetList.Datasets {
log.Print(dataset.DatasetReference.DatasetId)
}
/*
DataSet Get
*/
Dataset, _ := service.Datasets.Get(projectId, datasetId).Do()
if Dataset != nil {
log.Print(Dataset.DatasetReference.DatasetId)
} else {
log.Print("dataset not exist")
}
ds := bigquery.Dataset{
DatasetReference:&bigquery.DatasetReference{
ProjectId:projectId,
DatasetId:"test_dataset",
},
}
/*
Table List
*/
service.Datasets.Insert(projectId, &ds).Do()
TableList, _ := service.Tables.List(projectId, dataSetId).Do()
for _, tableListTables := range TableList.Tables {
log.Print(tableListTables.TableReference.TableId)
}
/*
Table Get
*/
tableListTables, _ := service.Tables.Get(
projectId,
dataSetId,
tableName).Do()
if tableListTables != nil {
log.Print(tableListTables.TableReference.TableId)
} else {
log.Print("table not exist")
}
/*
Make Table
*/
schema := CreateTableSchema(TableModel{})
table := bigquery.Table{
TableReference: &bigquery.TableReference{
DatasetId: dataSetId,
ProjectId: projectId,
TableId: tableName,
},
Schema:&schema,
}
tableRes, _ := service.Tables.Insert(
projectId,
dataSetId,
&table).Do()
log.Print(tableRes)
/*
InsertAll
*/
res, _ := service.Tabledata.InsertAll(
projectId,
dataSetId,
tableName,
&bigquery.TableDataInsertAllRequest{
Kind: "bigquery#tableDataInsertAllRequest",
Rows: CreateInsertRow(TableModel{Name:"name", Address:"address"}),
}).Do()
log.Print(res)
/*
Query
*/
query := "SELECT address FROM [<project-name>:<dataset-name>.<table-name>] LIMIT 1000"
result, err := service.Jobs.Query(projectId, &bigquery.QueryRequest{
Kind: "bigquery#queryRequest",
Query: query,
}).Do()
for _, row := range result.Rows {
log.Print(row)
for _, cell := range row.F {
log.Print(cell.V)
}
}
}
gist
go get でverisonを指定したいとき
git から直接バージョンを指定する方法で解決した。
git tag -l git checkout tags/v0.0.2
UbuntuへGo1.7をインストールする
UbuntuへGo1.7をインストールしたのでその時の作業ログ
$ sudo apt-get update $ sudo apt-get -y upgrade
$ wget https://storage.googleapis.com/golang/go1.7.4.linux-amd64.tar.gz $ sudo tar -xvf go1.7.4.linux-amd64.tar.gz $ sudo mv go /usr/local
$ mkdir ~/gocode
$ echo export GOROOT=/usr/local/go >> ~/.bash_profile $ echo export GOPATH=$HOME/gocode >> ~/.bash_profile $ echo export PATH=$GOPATH/bin:$GOROOT/bin:$PATH >> ~/.bash_profile $ source ~/.bash_profile
$ go version
versionが表示されればOK.
go version go1.7.4 linux/amd64
SupervisorでGoをデーモン化する。CentOS/Ubuntu
GoアプリケーションをCentOS環境で、デーモン化する必要があったので、その時の作業ログ。
CentOS
$ wget https://bootstrap.pypa.io/ez_setup.py -O - | python $ easy_install --version
$ easy_install supervisor $ supervisord -v
$ echo_supervisord_conf > /etc/supervisord.conf
設定ファイルを変更
;/etc/supervisord.conf [unix_http_server] file = /tmp/supervisor.sock chmod = 0777 chown= root:root [inet_http_server] port=9001 username = admin password = yourpassword [supervisorctl] serverurl = unix:///tmp/supervisor.sock [supervisord] logfile=/var/log/supervisord/supervisord.log logfile_maxbytes=50MB logfile_backups=10 loglevel=info pidfile=/var/run/supervisord.pid nodaemon=true minfds=1024 minprocs=200 user=root childlogdir=/var/log/supervisord/ [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface [program:app] command=/path-to-path/yourappname autostart = true startsecs = 5 user = root redirect_stderr = true stdout_logfile = /var/log/supervisord/app.log
Run
supervisord
Stop/Reload
以下のコマンドで管理モードに入る
supervisorctl reload
Ubuntu
$ sudo apt-get install supervisor
$ sudo vim /etc/supervisor/conf.d/app.conf
設定ファイルを変更
[unix_http_server] file=/var/run/supervisor.sock chmod=0700 [supervisord] logfile=/var/log/supervisor/supervisord.log pidfile=/var/run/supervisord.pid childlogdir=/var/log/supervisor [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface [supervisorctl] serverurl=unix:///var/run/supervisor.sock [include] files = /etc/supervisor/conf.d/*.conf [program:app] command=<your path> autostart=true autorestart=true startretries=10 user=root redirect_stderr=true stderr_logfile=/var/log/long.err.log stdout_logfile=/var/log/long.out.log stdout_logfile_maxbytes=50MB stdout_logfile_backups=10
(command に指定する前にbuildしておく必要がある。)
$ sudo service supervisor restart
GAE/GO datastoreのローカル開発をしてみる。
appengine/datastoreをローカルの開発環境でテストしたいときはどうするのかなと思ったけれど、以下でOKであった。
command
goapp test ./src/<path>
source
import (
"testing"
"google.golang.org/appengine"
"google.golang.org/appengine/aetest"
"google.golang.org/appengine/datastore"
)
func TestPostCompany(t *testing.T) {
inst, err := aetest.NewInstance(nil)
if err != nil {
t.Fatalf("Failed to create instance: %v", err)
}
defer inst.Close()
req1, err := inst.NewRequest("GET", "/gophers", nil)
if err != nil {
t.Fatalf("Failed to create req1: %v", err)
}
c1 := appengine.NewContext(req1)
var companyResponse = struct {
Name, Address, Nick, Key string
}{}
var key *datastore.Key
key, _ = datastore.DecodeKey(companyResponse.Key)
var savedCompany = companyResponse
datastore.Get(c1, key, &savedCompany)
datastore.Delete(c1, key)
}
link
Local Unit Testing for Go | App Engine standard environment for Go | Google Cloud Platform
しかし、おそいな、、もっと早くできなものか、、
