From 53e3c2e263b1d9c9f25dd1e30c18ad6bc04e05d3 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 5 Aug 2019 22:14:58 +0100 Subject: [PATCH] build: add azure pipelines build --- .gitattributes | 4 + Makefile | 11 ++- azure-pipelines.yml | 231 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 242 insertions(+), 4 deletions(-) create mode 100644 azure-pipelines.yml diff --git a/.gitattributes b/.gitattributes index 149fc873d..35157cfda 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,7 @@ # Ignore generated files in GitHub language statistics and diffs /MANUAL.* linguist-generated=true /rclone.1 linguist-generated=true + +# Don't fiddle with the line endings of test data +**/testdata/** -text +**/test/** -text diff --git a/Makefile b/Makefile index e1d22a53c..545b9d52e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL = bash -BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) +BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(BUILD_SOURCEBRANCHNAME),$(shell git rev-parse --abbrev-ref HEAD)) LAST_TAG := $(shell git describe --tags --abbrev=0) ifeq ($(BRANCH),$(LAST_TAG)) BRANCH := master @@ -17,7 +17,10 @@ ifneq ($(TAG),$(LAST_TAG)) endif GO_VERSION := $(shell go version) GO_FILES := $(shell go list ./... | grep -v /vendor/ ) -BETA_PATH := $(BRANCH_PATH)$(TAG) +ifdef BETA_SUBDIR + BETA_SUBDIR := /$(BETA_SUBDIR) +endif +BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR) BETA_URL := https://beta.rclone.org/$(BETA_PATH)/ BETA_UPLOAD_ROOT := memstore:beta-rclone-org BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH) @@ -168,14 +171,14 @@ endif @echo Beta release ready at $(BETA_URL)/testbuilds travis_beta: -ifeq ($(TRAVIS_OS_NAME),linux) +ifeq (linux,$(filter linux,$(subst Linux,linux,$(TRAVIS_OS_NAME) $(AGENT_OS)))) go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz' endif git log $(LAST_TAG).. > /tmp/git-log.txt go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG) rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD) ifndef BRANCH_PATH - rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT) + rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR) endif @echo Beta release ready at $(BETA_URL) diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..d85e02c72 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,231 @@ +--- +# Azure pipelines build for rclone +# Parts stolen shamelessly from all round the Internet, especially Caddy +# -*- compile-command: "yamllint -f parsable azure-pipelines.yml" -*- + +trigger: + branches: + include: + - '*' + tags: + include: + - '*' + +variables: + GOROOT: $(gorootDir)/go + GOPATH: $(system.defaultWorkingDirectory)/gopath + GOCACHE: $(system.defaultWorkingDirectory)/gocache + GOBIN: $(GOPATH)/bin + modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' + GO111MODULE: 'off' + GOTAGS: cmount + GO_LATEST: false + CPATH: '' + GO_INSTALL_ARCH: amd64 + +strategy: + matrix: + linux: + imageName: ubuntu-16.04 + gorootDir: /usr/local + GO_VERSION: latest + GOTAGS: cmount + BUILD_FLAGS: '-include "^linux/"' + MAKE_CHECK: true + MAKE_QUICKTEST: true + DEPLOY: true + mac: + imageName: macos-10.13 + gorootDir: /usr/local + GO_VERSION: latest + GOTAGS: "" # cmount doesn't work on osx travis for some reason + BUILD_FLAGS: '-include "^darwin/" -cgo' + MAKE_QUICKTEST: true + MAKE_RACEQUICKTEST: true + DEPLOY: true + windows_amd64: + imageName: windows-2019 + gorootDir: C:\ + GO_VERSION: latest + BUILD_FLAGS: '-include "^windows/amd64" -cgo' + MAKE_QUICKTEST: true + DEPLOY: true + windows_386: + imageName: windows-2019 + gorootDir: C:\ + GO_VERSION: latest + GO_INSTALL_ARCH: 386 + BUILD_FLAGS: '-include "^windows/386" -cgo' + MAKE_QUICKTEST: true + DEPLOY: true + other_os: + imageName: ubuntu-16.04 + gorootDir: /usr/local + GO_VERSION: latest + BUILD_FLAGS: '-exclude "^(windows|darwin|linux)/"' + MAKE_COMPILE_ALL: true + DEPLOY: true + modules_race: + imageName: ubuntu-16.04 + gorootDir: /usr/local + GO_VERSION: latest + GO111MODULE: on + GOPROXY: https://proxy.golang.org + MAKE_QUICKTEST: true + MAKE_RACEQUICKTEST: true + go1.9: + imageName: ubuntu-16.04 + gorootDir: /usr/local + GOCACHE: '' # build caching only came in go1.10 + GO_VERSION: go1.9.7 + MAKE_QUICKTEST: true + go1.10: + imageName: ubuntu-16.04 + gorootDir: /usr/local + GO_VERSION: go1.10.8 + MAKE_QUICKTEST: true + go1.11: + imageName: ubuntu-16.04 + gorootDir: /usr/local + GO_VERSION: go1.11.12 + MAKE_QUICKTEST: true + +pool: + vmImage: $(imageName) + +steps: + - bash: | + latestGo=$(curl "https://golang.org/VERSION?m=text") + echo "##vso[task.setvariable variable=GO_VERSION]$latestGo" + echo "##vso[task.setvariable variable=GO_LATEST]true" + echo "Latest Go version: $latestGo" + condition: eq( variables['GO_VERSION'], 'latest' ) + displayName: "Get latest Go version" + + - bash: | + sudo rm -f $(which go) + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + mkdir -p '$(modulePath)' + shopt -s extglob + shopt -s dotglob + mv !(gopath) '$(modulePath)' + displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH + + - task: CacheBeta@0 + continueOnError: true + inputs: + key: go-build-cache | "$(Agent.JobName)" + path: $(GOCACHE) + displayName: Cache go build + condition: ne( variables['GOCACHE'], '' ) + + # Install Libraries (varies by platform) + + - bash: | + sudo modprobe fuse + sudo chmod 666 /dev/fuse + sudo chown root:$USER /etc/fuse.conf + sudo apt-get install fuse libfuse-dev rpm pkg-config + condition: eq( variables['Agent.OS'], 'Linux' ) + displayName: Install Libraries on Linux + + - bash: | + brew update + brew tap caskroom/cask + brew cask install osxfuse + condition: eq( variables['Agent.OS'], 'Darwin' ) + displayName: Install Libraries on macOS + + - powershell: | + $ProgressPreference = 'SilentlyContinue' + choco install -y winfsp zip + Write-Host "##vso[task.setvariable variable=CPATH]C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" + if ($env:GO_INSTALL_ARCH -eq "386") { + choco install -y mingw --forcex86 --force + Write-Host "##vso[task.prependpath]C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" + } + # Copy mingw32-make.exe to make.exe so the same command line + # can be used on Windows as on macOS and Linux + $path = (get-command mingw32-make.exe).Path + Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe') + condition: eq( variables['Agent.OS'], 'Windows_NT' ) + displayName: Install Libraries on Windows + + + # Install Go (this varies by platform) + + - bash: | + wget "https://dl.google.com/go/$(GO_VERSION).linux-$(GO_INSTALL_ARCH).tar.gz" + sudo mkdir $(gorootDir) + sudo chown ${USER}:${USER} $(gorootDir) + tar -C $(gorootDir) -xzf "$(GO_VERSION).linux-$(GO_INSTALL_ARCH).tar.gz" + condition: eq( variables['Agent.OS'], 'Linux' ) + displayName: Install Go on Linux + + - bash: | + wget "https://dl.google.com/go/$(GO_VERSION).darwin-$(GO_INSTALL_ARCH).tar.gz" + sudo tar -C $(gorootDir) -xzf "$(GO_VERSION).darwin-$(GO_INSTALL_ARCH).tar.gz" + condition: eq( variables['Agent.OS'], 'Darwin' ) + displayName: Install Go on macOS + + - powershell: | + $ProgressPreference = 'SilentlyContinue' + Write-Host "Downloading Go $(GO_VERSION) for $(GO_INSTALL_ARCH)" + (New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip", "$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip") + Write-Host "Extracting Go" + Expand-Archive "$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip" -DestinationPath "$(gorootDir)" + condition: eq( variables['Agent.OS'], 'Windows_NT' ) + displayName: Install Go on Windows + + # Display environment for debugging + + - bash: | + printf "Using go at: $(which go)\n" + printf "Go version: $(go version)\n" + printf "\n\nGo environment:\n\n" + go env + printf "\n\nRclone environment:\n\n" + make vars + printf "\n\nSystem environment:\n\n" + env + workingDirectory: '$(modulePath)' + displayName: Print Go version and environment + + # Run Tests + + - bash: | + make + make quicktest + workingDirectory: '$(modulePath)' + displayName: Run tests + condition: eq( variables['MAKE_QUICKTEST'], 'true' ) + + - bash: | + make racequicktest + workingDirectory: '$(modulePath)' + displayName: Race test + condition: eq( variables['MAKE_RACEQUICKTEST'], 'true' ) + + - bash: | + make build_dep + make check + workingDirectory: '$(modulePath)' + displayName: Code quality test + condition: eq( variables['MAKE_CHECK'], 'true' ) + + - bash: | + make + make compile_all + workingDirectory: '$(modulePath)' + displayName: Compile all architectures test + condition: eq( variables['MAKE_COMPILE_ALL'], 'true' ) + + - bash: | + make travis_beta + env: + RCLONE_CONFIG_PASS: $(RCLONE_CONFIG_PASS) + BETA_SUBDIR: 'azure_pipelines' # FIXME remove when removing travis/appveyor + workingDirectory: '$(modulePath)' + displayName: Deploy built binaries + condition: and( eq( variables['DEPLOY'], 'true' ), ne( variables['Build.Reason'], 'PullRequest' ) )