[Golang Gzip]

Tags: #golang #go #gzip #proxy

Golang A basic Gzip read using data from HTTPBin.go

package main

import (
	"compress/gzip"
	"io"
	"log"
	"net/http"
	"os"
	"time"
)

func main() {
	client := &http.Client{
		Timeout: time.Second * time.Duration(5*time.Second),
	}

	req, err := http.NewRequest("GET", "https://httpbin.org/gzip", nil)
	if err != nil {
		log.Fatalf("http new request error: %s", err)
	}
  
 	// NOTE: If Accept-Encoding isn't presented to httpbin server, it won't send gzip response.
	req.Header.Add("Accept-Encoding", "gzip, deflate, br")

	resp, err := client.Do(req)
	if err != nil {
		log.Fatalf("http error: %s", err)
	}

	r, err := gzip.NewReader(resp.Body)
	if err != nil {
		log.Fatalf("new reader error: %s", err)
	}
	defer r.Close()

	io.Copy(os.Stdout, r)
}

Golang A basic Write and Read Gzip.go

package main

import (
	"bytes"
	"compress/gzip"
	"io/ioutil"
)

func main() {
	data := []byte("HelloWorld")

	var b bytes.Buffer
	w := gzip.NewWriter(&b)
	w.Write(data)
	w.Flush()
	w.Close()

	r, _ := gzip.NewReader(&b)
	defer r.Close()
	rawData, err := ioutil.ReadAll(r)
	println(string(rawData))
	if err != nil {
		panic(err.Error())
	}

}

Golang Accept Read from upstream and Write Gzip for downstream via Middleware.go

package main

import (
        "net/http"
        "compress/gzip"
        "io/ioutil"
        "strings"
        "sync"
        "io"
)

var gzPool = sync.Pool {
        New: func() interface{} {
                w := gzip.NewWriter(ioutil.Discard)
                return w
        },
}

type gzipResponseWriter struct {
        io.Writer
        http.ResponseWriter
}

func (w *gzipResponseWriter) WriteHeader(status int) {
        w.Header().Del("Content-Length")
        w.ResponseWriter.WriteHeader(status)
}

func (w *gzipResponseWriter) Write(b []byte) (int, error) {
        return w.Writer.Write(b)
}

func Gzip(next http.Handler) http.Handler {
        return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
                if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
                        next.ServeHTTP(w, r)
                        return
                }

                w.Header().Set("Content-Encoding", "gzip")

                gz := gzPool.Get().(*gzip.Writer)
                defer gzPool.Put(gz)

                gz.Reset(w)
                defer gz.Close()

                next.ServeHTTP(&gzipResponseWriter{ResponseWriter: w, Writer: gz}, r)
        })
}

Golang Write and Read Gzip via HTTP Reverse Proxy.go

// https://github.com/golang/go/issues/14975
// slightly broken in the sense of not handling gz.Close()
//
// https://gist.github.com/erikdubbelboer/7df2b2b9f34f9f839a84
// updated fork with fix + other refactorings + tests

package main

import (
    "compress/gzip"
    "fmt"
    "io"
    "io/ioutil"
    "net/http"
    "net/http/httputil"
    "net/url"
    "strings"
    "time"
)

// Gzip from https://gist.github.com/the42/1956518
type gzipResponseWriter struct {
    io.Writer
    http.ResponseWriter
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
    return w.Writer.Write(b)
}

func (w gzipResponseWriter) WriteHeader(code int) {
    fmt.Printf("Writing header: %v\n", code)
    w.Header().Del("Content-Length")
    w.ResponseWriter.WriteHeader(code)
}

func makeGzipHandler(fn http.HandlerFunc) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
            fn(w, r)
            return
        }
        w.Header().Set("Content-Encoding", "gzip")
        gz := gzip.NewWriter(w)
        defer func() {
            err := gz.Close()
            if err != nil {
                fmt.Printf("Error closing gzip: %+v\n", err)
            }
        }()
        gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
        fn(gzr, r)
    }
}

// Handler that does not set a content length, so, golang uses chunking.
func handler(w http.ResponseWriter, r *http.Request) {
    message := "Hello, world!"
    w.Header().Set("Content-Type", "text/plain")
    w.Write([]byte(message))
}

// Constructs a reverse proxy to the given port.
func reverseProxy(port string) func(http.ResponseWriter, *http.Request) {
    url, err := url.Parse("http://127.0.0.1" + port)
    if err != nil {
        panic(err)
    }
    return httputil.NewSingleHostReverseProxy(url).ServeHTTP
}

// Gets the content from the given server, then returns the error from reading the body.
func get(server http.Server) error {
    resp, err := http.Get("http://127.0.0.1" + server.Addr)
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()
    _, err = ioutil.ReadAll(resp.Body)
    return err
}

func main() {
    server := http.Server{
        Addr:    ":2000",
        Handler: http.HandlerFunc(handler),
    }
    go server.ListenAndServe()

    proxyServer := http.Server{
        Addr:    ":4000",
        Handler: makeGzipHandler(reverseProxy(server.Addr)),
    }
    go proxyServer.ListenAndServe()

    time.Sleep(10 * time.Millisecond)

    fmt.Printf("Server err: %v\n", get(server))
    fmt.Printf("Proxy server err: %v\n", get(proxyServer))
}

Golang fake repeat bytes and convert []byte to reader.go

package main

import (
	"bytes"
	"compress/gzip"
  	"encoding/base64"
	"fmt"
	"io/ioutil"
    "math/rand"
)

func main() {
	var b bytes.Buffer
	w := gzip.NewWriter(&b)
	w.Write(bytes.Repeat([]byte("x"), 800))
	w.Flush()
	w.Close()

	b1 := bytes.NewReader(b.Bytes())
	fmt.Printf("%+v (%T)\n", b1, b1)

	r, _ := gzip.NewReader(&b)
	defer r.Close()
	rawData, err := ioutil.ReadAll(r)
	println(string(rawData))
	if err != nil {
		panic(err.Error())
	}

  	// use "math/rand"'s rand.Read to populate []byte with random data of specified length
  	genBytes := make([]byte, 800)
	rand.Read(genBytes) // generally won't be readable so might _look_ compressed even when decompressed
						// so maybe write encoded data to the gzip writer (see below for example).

	var mockWriter bytes.Buffer
	w := gzip.NewWriter(&mockWriter)
	w.Write(genBytes)
	w.Flush()
	w.Close()
	
  	debugReadCloser := ioutil.NopCloser(bytes.NewReader(mockWriter.Bytes()))
  
  	// encoded example
  
  	genBytes := make([]byte, 406715)
	rand.Read(genBytes)

	encoded := make([]byte, 542288)
	base64.StdEncoding.Encode(encoded, genBytes)

	var mockWriter bytes.Buffer
	w := gzip.NewWriter(&mockWriter)
	w.Write(encoded)
	w.Flush()
	w.Close()

	debugReadCloser := ioutil.NopCloser(bytes.NewReader(mockWriter.Bytes()))
}