Skip to content

Commit

Permalink
Migate tests to fit new Postgres system
Browse files Browse the repository at this point in the history
  • Loading branch information
@Tetrino
Tetrino committed Sep 5, 2023
1 parent a7a3117 commit cff0771
Show file tree
Hide file tree
Showing 37 changed files with 7,641 additions and 230 deletions.
31 changes: 25 additions & 6 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,33 @@ lint:
unit_tests:
go test -race $$(go list ./... | grep -v integration_tests)

integration_tests: build start_mongo
integration_tests: build start_postgres init_test_db
go test -race -v ./integration_tests

start_mongo:
./mongo.sh start

stop_mongo:
./mongo.sh stop
start_postgres:
docker run \
--name router-postgres-test-db \
-e POSTGRES_HOST_AUTH_METHOD=trust \
-d \
-p 5432:5432 \
--user 'postgres' \
--health-cmd 'pg_isready' \
--health-start-period 5s \
postgres:14
@echo Waiting for postgres to be up
@until [ "`docker inspect -f '{{.State.Health.Status}}' router-postgres-test-db`" = "healthy" ]; do \
echo '.\c' ; \
sleep 1 ; \
done ;

init_local_db:
docker exec -i govuk-docker_postgres-14_1 psql < localdb_init.sql

init_test_db:
docker exec -i router-postgres-test-db psql < localdb_init.sql

cleanup_postgres:
@docker rm -f router-postgres-test-db || true

update_deps:
go get -t -u ./... && go mod tidy && go mod vendor
73 changes: 49 additions & 24 deletions integration_tests/route_helpers.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package integration

import (
"database/sql"
"fmt"
"os"
"time"

"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
_ "github.com/lib/pq" // Without which we can't use PSQL calls

// revive:disable:dot-imports
. "github.com/onsi/ginkgo/v2"
Expand All @@ -19,18 +19,18 @@ var _ = AfterEach(func() {
})

var (
routerDB *mgo.Database
routerDB *sql.DB
)

type Route struct {
IncomingPath string `bson:"incoming_path"`
RouteType string `bson:"route_type"`
Handler string `bson:"handler"`
BackendID string `bson:"backend_id"`
RedirectTo string `bson:"redirect_to"`
RedirectType string `bson:"redirect_type"`
SegmentsMode string `bson:"segments_mode"`
Disabled bool `bson:"disabled"`
IncomingPath string
RouteType string
Handler string
BackendID string
RedirectTo string
RedirectType string
SegmentsMode string
Disabled bool
}

func NewBackendRoute(backendID string, extraParams ...string) Route {
Expand Down Expand Up @@ -80,36 +80,61 @@ func NewGoneRoute(extraParams ...string) Route {
}

func initRouteHelper() error {
databaseURL := os.Getenv("ROUTER_MONGO_URL")
databaseURL := os.Getenv("DATABASE_URL")

if databaseURL == "" {
databaseURL = "127.0.0.1"
databaseURL = "postgresql://postgres@127.0.0.1:5432/router_test?sslmode=disable"
}

sess, err := mgo.Dial(databaseURL)
db, err := sql.Open("postgres", databaseURL)
if err != nil {
return fmt.Errorf("failed to connect to mongo: %w", err)
return fmt.Errorf("Failed to connect to Postgres: " + err.Error())
}
sess.SetSyncTimeout(10 * time.Minute)
sess.SetSocketTimeout(10 * time.Minute)

routerDB = sess.DB("router_test")
db.SetConnMaxLifetime(10 * time.Minute)
db.SetMaxIdleConns(0)
db.SetMaxOpenConns(10)

routerDB = db
return nil
}

func addBackend(id, url string) {
err := routerDB.C("backends").Insert(bson.M{"backend_id": id, "backend_url": url})
Expect(err).NotTo(HaveOccurred())
query := `
INSERT INTO backends (backend_id, backend_url, created_at, updated_at)
VALUES ($1, $2, $3, $4)
`

_, err := routerDB.Exec(query, id, url, time.Now(), time.Now())
Expect(err).ToNot(HaveOccurred())
}

func addRoute(path string, route Route) {
route.IncomingPath = path

err := routerDB.C("routes").Insert(route)
Expect(err).NotTo(HaveOccurred())
query := `
INSERT INTO routes (incoming_path, route_type, handler, backend_id, redirect_to, redirect_type, segments_mode, disabled, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
`

_, err := routerDB.Exec(
query,
route.IncomingPath,
route.RouteType,
route.Handler,
route.BackendID,
route.RedirectTo,
route.RedirectType,
route.SegmentsMode,
route.Disabled,
time.Now(),
time.Now(),
)

Expect(err).ToNot(HaveOccurred())
}

func clearRoutes() {
_ = routerDB.C("routes").DropCollection()
_ = routerDB.C("backends").DropCollection()
_, err := routerDB.Exec("DELETE FROM routes; DELETE FROM backends")
Expect(err).ToNot(HaveOccurred())
}
6 changes: 3 additions & 3 deletions integration_tests/router_support.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ import (
)

const (
routerPort = 3169
apiPort = 3168
routerPort = 5434
apiPort = 5433
)

func routerURL(port int, path string) string {
Expand Down Expand Up @@ -56,7 +56,7 @@ func startRouter(port, apiPort int, extraEnv []string) error {
}
cmd := exec.Command(bin)

cmd.Env = append(cmd.Environ(), "ROUTER_MONGO_DB=router_test")
cmd.Env = append(cmd.Environ(), "DATABASE_NAME=router_test")
cmd.Env = append(cmd.Env, fmt.Sprintf("ROUTER_PUBADDR=%s", pubAddr))
cmd.Env = append(cmd.Env, fmt.Sprintf("ROUTER_APIADDR=%s", apiAddr))
cmd.Env = append(cmd.Env, fmt.Sprintf("ROUTER_ERROR_LOG=%s", tempLogfile.Name()))
Expand Down
70 changes: 34 additions & 36 deletions lib/router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,20 +56,20 @@ type Options struct {
}

type Backend struct {
BackendID string `bson:"backend_id"`
BackendURL string `bson:"backend_url"`
SubdomainName string `bson:"subdomain_name"`
BackendID sql.NullString
BackendURL sql.NullString
SubdomainName sql.NullString
}

type Route struct {
IncomingPath string `bson:"incoming_path"`
RouteType string `bson:"route_type"`
Handler string `bson:"handler"`
BackendID string `bson:"backend_id"`
RedirectTo string `bson:"redirect_to"`
RedirectType string `bson:"redirect_type"`
SegmentsMode string `bson:"segments_mode"`
Disabled bool `bson:"disabled"`
IncomingPath sql.NullString
RouteType sql.NullString
Handler sql.NullString
BackendID sql.NullString
RedirectTo sql.NullString
RedirectType sql.NullString
SegmentsMode sql.NullString
Disabled bool
}

// RegisterMetrics registers Prometheus metrics from the router module and the
Expand Down Expand Up @@ -102,7 +102,7 @@ func NewRouter(o Options) (rt *Router, err error) {
listener := pq.NewListener(o.DatabaseURL, 10*time.Second, time.Minute, listenerProblemReporter)
o.Listener = listener

err = listener.Listen("events")
err = listener.Listen("notify")
if err != nil {
panic(err)
}
Expand Down Expand Up @@ -185,7 +185,7 @@ func (rt *Router) pollAndReload() {
logDebug("router: updates found")
rt.reloadRoutes(sess)
} else {
logDebug("router: no updates found")
logDebug("router: no updates found - really?")
}
}()
}
Expand All @@ -210,8 +210,6 @@ func (rt *Router) reloadRoutes(db *sql.DB) {
errorMessage := fmt.Sprintf("panic: %v", r)
err := logger.RecoveredError{ErrorMessage: errorMessage}
logger.NotifySentry(logger.ReportableError{Error: err})

routeReloadErrorCountMetric.Inc()
}
timer.ObserveDuration()
}()
Expand Down Expand Up @@ -252,7 +250,7 @@ func (rt *Router) loadBackends(db *sql.DB) (backends map[string]http.Handler) {
backend := &Backend{}
backends = make(map[string]http.Handler)

rows, err := db.Query("SELECT * FROM backends")
rows, err := db.Query("SELECT backend_id, backend_url FROM backends")
if err != nil {
logWarn(fmt.Sprintf("pq: error retrieving row information from table, skipping update. (error: %v)", err))
return
Expand All @@ -268,12 +266,12 @@ func (rt *Router) loadBackends(db *sql.DB) (backends map[string]http.Handler) {
backendURL, err := backend.ParseURL()
if err != nil {
logWarn(fmt.Sprintf("router: couldn't parse URL %s for backends %s "+
"(error: %v), skipping!", backend.BackendURL, backend.BackendID, err))
"(error: %v), skipping!", backend.BackendURL.String, backend.BackendID.String, err))
continue
}

backends[backend.BackendID] = handlers.NewBackendHandler(
backend.BackendID,
backends[backend.BackendID.String] = handlers.NewBackendHandler(
backend.BackendID.String,
backendURL,
rt.opts.BackendConnTimeout,
rt.opts.BackendHeaderTimeout,
Expand All @@ -289,7 +287,7 @@ func (rt *Router) loadBackends(db *sql.DB) (backends map[string]http.Handler) {
func loadRoutes(db *sql.DB, mux *triemux.Mux, backends map[string]http.Handler) {
route := &Route{}

rows, err := db.Query("SELECT * FROM routes")
rows, err := db.Query("SELECT incoming_path, route_type, handler, disabled, backend_id, redirect_to, redirect_type, segments_mode FROM routes")
if err != nil {
logWarn(fmt.Sprintf("pq: error retrieving row information from table, skipping update. (error: %v)", err))
return
Expand All @@ -308,13 +306,13 @@ func loadRoutes(db *sql.DB, mux *triemux.Mux, backends map[string]http.Handler)
logWarn(fmt.Sprintf("pq: error retrieving row information from table, skipping update. (error: %v)", err))
return
}
prefix := (route.RouteType == RouteTypePrefix)
prefix := (route.RouteType.String == RouteTypePrefix)

// the database contains paths with % encoded routes.
// Unescape them here because the http.Request objects we match against contain the unescaped variants.
incomingURL, err := url.Parse(route.IncomingPath)
incomingURL, err := url.Parse(route.IncomingPath.String)
if err != nil {
logWarn(fmt.Sprintf("router: found route %+v with invalid incoming path '%s', skipping!", route, route.IncomingPath))
logWarn(fmt.Sprintf("router: found route %+v with invalid incoming path '%s', skipping!", route, route.IncomingPath.String))
continue
}

Expand All @@ -324,23 +322,23 @@ func loadRoutes(db *sql.DB, mux *triemux.Mux, backends map[string]http.Handler)
continue
}

switch route.Handler {
switch route.Handler.String {
case "backend":
handler, ok := backends[route.BackendID]
handler, ok := backends[route.BackendID.String]
if !ok {
logWarn(fmt.Sprintf("router: found route %+v which references unknown backend "+
"%s, skipping!", route, route.BackendID))
"%s, skipping!", route, route.BackendID.String))
continue
}
mux.Handle(incomingURL.Path, prefix, handler)
logDebug(fmt.Sprintf("router: registered %s (prefix: %v) for %s",
incomingURL.Path, prefix, route.BackendID))
incomingURL.Path, prefix, route.BackendID.String))
case "redirect":
redirectTemporarily := (route.RedirectType == "temporary")
handler := handlers.NewRedirectHandler(incomingURL.Path, route.RedirectTo, shouldPreserveSegments(route), redirectTemporarily)
redirectTemporarily := (route.RedirectType.String == "temporary")
handler := handlers.NewRedirectHandler(incomingURL.Path, route.RedirectTo.String, shouldPreserveSegments(route), redirectTemporarily)
mux.Handle(incomingURL.Path, prefix, handler)
logDebug(fmt.Sprintf("router: registered %s (prefix: %v) -> %s",
incomingURL.Path, prefix, route.RedirectTo))
incomingURL.Path, prefix, route.RedirectTo.String))
case "gone":
mux.Handle(incomingURL.Path, prefix, goneHandler)
logDebug(fmt.Sprintf("router: registered %s (prefix: %v) -> Gone", incomingURL.Path, prefix))
Expand All @@ -352,26 +350,26 @@ func loadRoutes(db *sql.DB, mux *triemux.Mux, backends map[string]http.Handler)
logDebug(fmt.Sprintf("router: registered %s (prefix: %v) -> Boom!!!", incomingURL.Path, prefix))
default:
logWarn(fmt.Sprintf("router: found route %+v with unknown handler type "+
"%s, skipping!", route, route.Handler))
"%s, skipping!", route, route.Handler.String))
continue
}
}
}

func (be *Backend) ParseURL() (*url.URL, error) {
backendURL := os.Getenv(fmt.Sprintf("BACKEND_URL_%s", be.BackendID))
backendURL := os.Getenv(fmt.Sprintf("BACKEND_URL_%s", be.BackendID.String))
if backendURL == "" {
backendURL = be.BackendURL
backendURL = be.BackendURL.String
}
return url.Parse(backendURL)
}

func shouldPreserveSegments(route *Route) bool {
switch route.RouteType {
switch route.RouteType.String {
case RouteTypeExact:
return route.SegmentsMode == SegmentsModePreserve
return route.SegmentsMode.String == SegmentsModePreserve
case RouteTypePrefix:
return route.SegmentsMode != SegmentsModeIgnore
return route.SegmentsMode.String != SegmentsModeIgnore
default:
return false
}
Expand Down
Loading

0 comments on commit cff0771

Please sign in to comment.