webhdfs
Examples
- More examples can be found in client_test.go.
- Write the file to HDFS and read the file back:
package main
import (
"io/ioutil"
"log"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/searKing/webhdfs"
)
const (
webHdfsEndpoint = "quickstart.cloudera:50070"
KerberosRealm = "CLOUDERA"
KerberosUsername = "hdfs/quickstart.cloudera"
KerberosServicePrincipleName = "HTTP/quickstart.cloudera"
KerberosConfigFile = "/etc/krb5.conf"
// pick one auth way by 3 ways below, if all below are empty, kerberos is disabled.
KerberosPassword = ""
KerberosKeyTabFile = "/hdfs.keytab" // /krb5.keytab
KerberosCCacheFile = "/tmp/krb5cc_0" // /tmp/krb5cc_0
)
func getClient() *webhdfs.Client {
c, err := webhdfs.New(webHdfsEndpoint, webhdfs.WithDisableSSL(true),
webhdfs.WithKerberosKeytabFile(KerberosUsername, KerberosServicePrincipleName, KerberosRealm, KerberosKeyTabFile, KerberosConfigFile))
if err != nil {
log.Fatalf("create client %s", err)
}
return c
}
func main() {
file := "/data/test/sample.txt"
{ // upload from hdfs
resp, err := getClient().Create(&webhdfs.CreateRequest{
Path: aws.String(file),
Body: strings.NewReader("test_input"),
Overwrite: aws.Bool(true),
})
if err != nil {
log.Fatalf("webhdfs Open failed: %s", err)
}
defer resp.Body.Close()
}
{ // download from hdfs
resp, err := getClient().Open(&webhdfs.OpenRequest{
Path: aws.String(file),
})
if err != nil {
t.Fatalf("webhdfs Open failed: %s", err)
}
defer resp.Body.Close()
content, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalf("webhdfs Read failed: %s", err)
}
log.Printf("content: %s", string(content))
}
}
TODO
- Improve documentation
- More examples