mirror of
https://github.com/rclone/rclone
synced 2024-11-01 21:49:35 +01:00
mount: add test scripts
This commit is contained in:
Nick Craig-Wood
parent
990b676e13
commit
3ef9f6f016
72
cmd/mount/test/seek_speed.go
Normal file
72
cmd/mount/test/seek_speed.go
Normal file
@ -0,0 +1,72 @@
|
||||
// +build ignore
|
||||
|
||||
// Read blocks out of a single file to time the seeking code
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// Flags
|
||||
iterations = flag.Int("n", 25, "Iterations to try")
|
||||
maxBlockSize = flag.Int("b", 1024*1024, "Max block size to read")
|
||||
randSeed = flag.Int64("seed", 1, "Seed for the random number generator")
|
||||
)
|
||||
|
||||
func randomSeekTest(size int64, in *os.File, name string) {
|
||||
start := rand.Int63n(size)
|
||||
blockSize := rand.Intn(*maxBlockSize)
|
||||
if int64(blockSize) > size-start {
|
||||
blockSize = int(size - start)
|
||||
}
|
||||
log.Printf("Reading %d from %d", blockSize, start)
|
||||
|
||||
_, err := in.Seek(start, 0)
|
||||
if err != nil {
|
||||
log.Fatalf("Seek failed on %q: %v", name, err)
|
||||
}
|
||||
|
||||
buf := make([]byte, blockSize)
|
||||
_, err = io.ReadFull(in, buf)
|
||||
if err != nil {
|
||||
log.Fatalf("Read failed on %q: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
args := flag.Args()
|
||||
if len(args) != 1 {
|
||||
log.Fatalf("Require 1 file as argument")
|
||||
}
|
||||
rand.Seed(*randSeed)
|
||||
|
||||
name := args[0]
|
||||
in, err := os.Open(name)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't open %q: %v", name, err)
|
||||
}
|
||||
|
||||
fi, err := in.Stat()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't stat %q: %v", name, err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
for i := 0; i < *iterations; i++ {
|
||||
randomSeekTest(fi.Size(), in, name)
|
||||
}
|
||||
dt := time.Since(start)
|
||||
log.Printf("That took %v for %d iterations, %v per iteration", dt, *iterations, dt/time.Duration(*iterations))
|
||||
|
||||
err = in.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("Error closing %q: %v", name, err)
|
||||
}
|
||||
}
|
||||
115
cmd/mount/test/seeker.go
Normal file
115
cmd/mount/test/seeker.go
Normal file
@ -0,0 +1,115 @@
|
||||
// +build ignore
|
||||
|
||||
// Read two files with lots of seeking to stress test the seek code
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// Flags
|
||||
iterations = flag.Int("n", 1E6, "Iterations to try")
|
||||
maxBlockSize = flag.Int("b", 1024*1024, "Max block size to read")
|
||||
)
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func randomSeekTest(size int64, in1, in2 *os.File, file1, file2 string) {
|
||||
start := rand.Int63n(size)
|
||||
blockSize := rand.Intn(*maxBlockSize)
|
||||
if int64(blockSize) > size-start {
|
||||
blockSize = int(size - start)
|
||||
}
|
||||
log.Printf("Reading %d from %d", blockSize, start)
|
||||
|
||||
_, err := in1.Seek(start, 0)
|
||||
if err != nil {
|
||||
log.Fatalf("Seek failed on %q: %v", file1, err)
|
||||
}
|
||||
_, err = in2.Seek(start, 0)
|
||||
if err != nil {
|
||||
log.Fatalf("Seek failed on %q: %v", file2, err)
|
||||
}
|
||||
|
||||
buf1 := make([]byte, blockSize)
|
||||
n1, err := io.ReadFull(in1, buf1)
|
||||
if err != nil {
|
||||
log.Fatalf("Read failed on %q: %v", file1, err)
|
||||
}
|
||||
|
||||
buf2 := make([]byte, blockSize)
|
||||
n2, err := io.ReadFull(in2, buf2)
|
||||
if err != nil {
|
||||
log.Fatalf("Read failed on %q: %v", file2, err)
|
||||
}
|
||||
|
||||
if n1 != n2 {
|
||||
log.Fatalf("Read different lengths %d (%q) != %d (%q)", n1, file1, n2, file2)
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf1, buf2) {
|
||||
log.Printf("Dumping different blocks")
|
||||
err = ioutil.WriteFile("/tmp/z1", buf1, 0777)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to write /tmp/z1: %v", err)
|
||||
}
|
||||
err = ioutil.WriteFile("/tmp/z2", buf2, 0777)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to write /tmp/z2: %v", err)
|
||||
}
|
||||
log.Fatalf("Read different contents - saved in /tmp/z1 and /tmp/z2")
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
args := flag.Args()
|
||||
if len(args) != 2 {
|
||||
log.Fatalf("Require 2 files as argument")
|
||||
}
|
||||
file1, file2 := args[0], args[1]
|
||||
in1, err := os.Open(file1)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't open %q: %v", file1, err)
|
||||
}
|
||||
in2, err := os.Open(file2)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't open %q: %v", file2, err)
|
||||
}
|
||||
|
||||
fi1, err := in1.Stat()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't stat %q: %v", file1, err)
|
||||
}
|
||||
fi2, err := in2.Stat()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't stat %q: %v", file2, err)
|
||||
}
|
||||
|
||||
if fi1.Size() != fi2.Size() {
|
||||
log.Fatalf("Files not the same size")
|
||||
}
|
||||
|
||||
for i := 0; i < *iterations; i++ {
|
||||
randomSeekTest(fi1.Size(), in1, in2, file1, file2)
|
||||
}
|
||||
|
||||
err = in1.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("Error closing %q: %v", file1, err)
|
||||
}
|
||||
err = in2.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("Error closing %q: %v", file2, err)
|
||||
}
|
||||
}
|
||||
129
cmd/mount/test/seekers.go
Normal file
129
cmd/mount/test/seekers.go
Normal file
@ -0,0 +1,129 @@
|
||||
// +build ignore
|
||||
|
||||
// Read lots files with lots of simultaneous seeking to stress test the seek code
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// Flags
|
||||
iterations = flag.Int("n", 1E6, "Iterations to try")
|
||||
maxBlockSize = flag.Int("b", 1024*1024, "Max block size to read")
|
||||
simultaneous = flag.Int("transfers", 16, "Number of simultaneous files to open")
|
||||
seeksPerFile = flag.Int("seeks", 8, "Seeks per file")
|
||||
mask = flag.Int64("mask", 0, "mask for seek, eg 0x7fff")
|
||||
)
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func seekTest(n int, file string) {
|
||||
in, err := os.Open(file)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't open %q: %v", file, err)
|
||||
}
|
||||
fi, err := in.Stat()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't stat %q: %v", file, err)
|
||||
}
|
||||
size := fi.Size()
|
||||
|
||||
// FIXME make sure we try start and end
|
||||
|
||||
maxBlockSize := *maxBlockSize
|
||||
if int64(maxBlockSize) > size {
|
||||
maxBlockSize = int(size)
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
start := rand.Int63n(size)
|
||||
if *mask != 0 {
|
||||
start &^= *mask
|
||||
}
|
||||
blockSize := rand.Intn(maxBlockSize)
|
||||
beyondEnd := false
|
||||
switch rand.Intn(10) {
|
||||
case 0:
|
||||
start = 0
|
||||
case 1:
|
||||
start = size - int64(blockSize)
|
||||
case 2:
|
||||
// seek beyond the end
|
||||
start = size + int64(blockSize)
|
||||
beyondEnd = true
|
||||
default:
|
||||
}
|
||||
if !beyondEnd && int64(blockSize) > size-start {
|
||||
blockSize = int(size - start)
|
||||
}
|
||||
log.Printf("%s: Reading %d from %d", file, blockSize, start)
|
||||
|
||||
_, err = in.Seek(start, 0)
|
||||
if err != nil {
|
||||
log.Fatalf("Seek failed on %q: %v", file, err)
|
||||
}
|
||||
|
||||
buf := make([]byte, blockSize)
|
||||
n, err := io.ReadFull(in, buf)
|
||||
if beyondEnd && err == io.EOF {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
log.Fatalf("Read failed on %q: %v (%d)", file, err, n)
|
||||
}
|
||||
}
|
||||
|
||||
err = in.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("Error closing %q: %v", file, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Find all the files in dir
|
||||
func findFiles(dir string) (files []string) {
|
||||
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if info.Mode().IsRegular() && info.Size() > 0 {
|
||||
files = append(files, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(files)
|
||||
return files
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
args := flag.Args()
|
||||
if len(args) != 1 {
|
||||
log.Fatalf("Require a directory as argument")
|
||||
}
|
||||
dir := args[0]
|
||||
files := findFiles(dir)
|
||||
jobs := make(chan string, *simultaneous)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(*simultaneous)
|
||||
for i := 0; i < *simultaneous; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for file := range jobs {
|
||||
seekTest(*seeksPerFile, file)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for i := 0; i < *iterations; i++ {
|
||||
i := rand.Intn(len(files))
|
||||
jobs <- files[i]
|
||||
//jobs <- files[i]
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user