From 81c56218f31cad8bd5a0f9df6af339d10462356d Mon Sep 17 00:00:00 2001 From: Ashim Shrestha Date: Thu, 4 Apr 2019 07:10:11 -0700 Subject: [PATCH 1/7] update hackage mirror site --- hazel.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hazel.bzl b/hazel.bzl index 3f3ccd1..73cddda 100644 --- a/hazel.bzl +++ b/hazel.bzl @@ -15,7 +15,7 @@ def _cabal_haskell_repository_impl(ctx): ghc_workspace = get_ghc_workspace(ctx.attr.ghc_workspaces, ctx) pkg = "{}-{}".format(ctx.attr.package_name, ctx.attr.package_version) - url = "https://hackage.haskell.org/package/{}.tar.gz".format(pkg) + url = "https://hackage.fpcomplete.com/package/{}.tar.gz".format(pkg) # If the SHA is wrong, the error message is very unhelpful: # https://github.com/bazelbuild/bazel/issues/3709 # As a workaround, we compute it manually if it's not set (and then fail @@ -216,7 +216,7 @@ def hazel_custom_package_hackage( build_file and build_file_content are mutually exclusive. """ package_id = package_name + "-" + version - url = "https://hackage.haskell.org/package/{0}/{1}.tar.gz".format( + url = "https://hackage.fpcomplete.com/package/{0}/{1}.tar.gz".format( package_id, package_id, ) From 779acb2c1d5e727092cc4c83c6ad87cfe0ac7fcf Mon Sep 17 00:00:00 2001 From: Ashim Shrestha Date: Thu, 4 Apr 2019 07:14:49 -0700 Subject: [PATCH 2/7] http --- hazel.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hazel.bzl b/hazel.bzl index 73cddda..55d04ac 100644 --- a/hazel.bzl +++ b/hazel.bzl @@ -15,7 +15,7 @@ def _cabal_haskell_repository_impl(ctx): ghc_workspace = get_ghc_workspace(ctx.attr.ghc_workspaces, ctx) pkg = "{}-{}".format(ctx.attr.package_name, ctx.attr.package_version) - url = "https://hackage.fpcomplete.com/package/{}.tar.gz".format(pkg) + url = "http://hackage.fpcomplete.com/package/{}.tar.gz".format(pkg) # If the SHA is wrong, the error message is very unhelpful: # https://github.com/bazelbuild/bazel/issues/3709 # As a workaround, we compute it manually if it's not set (and then fail @@ -216,7 +216,7 @@ def hazel_custom_package_hackage( build_file and build_file_content are mutually exclusive. """ package_id = package_name + "-" + version - url = "https://hackage.fpcomplete.com/package/{0}/{1}.tar.gz".format( + url = "http://hackage.fpcomplete.com/package/{0}/{1}.tar.gz".format( package_id, package_id, ) From e1b2a9aa3cf9ec4236d511328ba88959b1051f1b Mon Sep 17 00:00:00 2001 From: Ashim Shrestha Date: Thu, 4 Apr 2019 07:27:33 -0700 Subject: [PATCH 3/7] another mirror --- hazel.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hazel.bzl b/hazel.bzl index 55d04ac..c0be09e 100644 --- a/hazel.bzl +++ b/hazel.bzl @@ -15,7 +15,7 @@ def _cabal_haskell_repository_impl(ctx): ghc_workspace = get_ghc_workspace(ctx.attr.ghc_workspaces, ctx) pkg = "{}-{}".format(ctx.attr.package_name, ctx.attr.package_version) - url = "http://hackage.fpcomplete.com/package/{}.tar.gz".format(pkg) + url = "http://objects-us-east-1.dream.io/hackage-mirror/package/{}.tar.gz".format(pkg) # If the SHA is wrong, the error message is very unhelpful: # https://github.com/bazelbuild/bazel/issues/3709 # As a workaround, we compute it manually if it's not set (and then fail @@ -216,7 +216,7 @@ def hazel_custom_package_hackage( build_file and build_file_content are mutually exclusive. """ package_id = package_name + "-" + version - url = "http://hackage.fpcomplete.com/package/{0}/{1}.tar.gz".format( + url = "http://objects-us-east-1.dream.io/hackage-mirror/package/{0}/{1}.tar.gz".format( package_id, package_id, ) From 56fcad0cfcb3436d44caca7a0de2185db8e9313f Mon Sep 17 00:00:00 2001 From: Ashim Shrestha Date: Thu, 4 Apr 2019 07:33:54 -0700 Subject: [PATCH 4/7] package location updates --- hazel.bzl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hazel.bzl b/hazel.bzl index c0be09e..9a0ead0 100644 --- a/hazel.bzl +++ b/hazel.bzl @@ -15,7 +15,7 @@ def _cabal_haskell_repository_impl(ctx): ghc_workspace = get_ghc_workspace(ctx.attr.ghc_workspaces, ctx) pkg = "{}-{}".format(ctx.attr.package_name, ctx.attr.package_version) - url = "http://objects-us-east-1.dream.io/hackage-mirror/package/{}.tar.gz".format(pkg) + url = "http://hackage.fpcomplete.com/package/{}.tar.gz".format(pkg) # If the SHA is wrong, the error message is very unhelpful: # https://github.com/bazelbuild/bazel/issues/3709 # As a workaround, we compute it manually if it's not set (and then fail @@ -216,8 +216,7 @@ def hazel_custom_package_hackage( build_file and build_file_content are mutually exclusive. """ package_id = package_name + "-" + version - url = "http://objects-us-east-1.dream.io/hackage-mirror/package/{0}/{1}.tar.gz".format( - package_id, + url = "http://hackage.fpcomplete.com/package/{}.tar.gz".format( package_id, ) if not build_file and not build_file_content: From 67f0e04603c42b60994b6f759297ac705582b410 Mon Sep 17 00:00:00 2001 From: Ashim Shrestha Date: Thu, 4 Apr 2019 07:46:02 -0700 Subject: [PATCH 5/7] stackage uri update --- Stackage.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Stackage.hs b/Stackage.hs index 439e136..29722a4 100755 --- a/Stackage.hs +++ b/Stackage.hs @@ -51,7 +51,7 @@ main = do Left err -> throw err Right (x :: BuildPlan) -> pure x shas <- flip Map.traverseWithKey (planPackages plan) $ \n p -> do - let hackageUrl = "https://hackage.haskell.org/package/" + let hackageUrl = "http://hackage.fpcomplete.com/package/" ++ Cabal.display n ++ "-" ++ Cabal.display (planPackageVersion p) ++ ".tar.gz" tar <- downloadUrl manager hackageUrl From c9a41545d3664420d08dfddb235c43eaa720e342 Mon Sep 17 00:00:00 2001 From: palmerlao Date: Thu, 4 Apr 2019 10:51:48 -0700 Subject: [PATCH 6/7] Make Stackage.hs try many URLs --- Stackage.hs | 50 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/Stackage.hs b/Stackage.hs index 29722a4..3f7700a 100755 --- a/Stackage.hs +++ b/Stackage.hs @@ -14,6 +14,7 @@ import qualified Crypto.Hash.SHA256 as SHA256 import Control.Monad (forM_, unless) import Data.Aeson.Types import Data.Bifunctor +import Data.List (intercalate) import Data.Yaml import Distribution.Package import Distribution.PackageDescription (FlagName, mkFlagName, unFlagName) @@ -31,6 +32,7 @@ import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as L import qualified Data.ByteString.Base16 as Base16 import qualified Data.ByteString.Char8 as BC +import qualified Data.List.NonEmpty as Nel import qualified Data.Map.Strict as Map import qualified Data.Text as T import qualified Distribution.Text as Cabal @@ -46,15 +48,17 @@ main = do "https://raw.githubusercontent.com/fpco/lts-haskell/master/" ++ resolver ++ ".yaml" _ -> "https://raw.githubusercontent.com/fpco/stackage-nightly/master/" ++ resolver ++ ".yaml" - ltsYaml <- downloadUrl manager ltsUrl + ltsYaml <- downloadUrl manager (ltsUrl Nel.:| []) plan <- case decodeEither' $ L.toStrict ltsYaml of Left err -> throw err Right (x :: BuildPlan) -> pure x shas <- flip Map.traverseWithKey (planPackages plan) $ \n p -> do - let hackageUrl = "http://hackage.fpcomplete.com/package/" - ++ Cabal.display n ++ "-" ++ Cabal.display (planPackageVersion p) - ++ ".tar.gz" - tar <- downloadUrl manager hackageUrl + let + tarball = Cabal.display n ++ "-" ++ Cabal.display (planPackageVersion p) ++ ".tar.gz" + fpcoUrl = "http://hackage.fpcomplete.com/package/" + hackageUrl = "http://hackage.haskell.org/package/" + urls = fmap (++tarball) (fpcoUrl Nel.:| [hackageUrl]) + tar <- downloadUrl manager urls putStrLn (Cabal.display n) return $! SHA256.hashlazy tar writeFile out $ show $ @@ -88,16 +92,32 @@ flagsExpr :: Flags -> Expr flagsExpr m = ExprDict $ bimap (ExprString . unFlagName) ExprBool <$> Map.toList m -downloadUrl :: Manager -> String -> IO L.ByteString -downloadUrl manager url = do - req <- parseRequest url - resp <- httpLbs req manager - let status = responseStatus resp - unless (statusIsSuccessful status) - $ error $ "Unable to download " ++ show url - ++ "\nStatus: " ++ show (statusCode status) - ++ " " ++ BC.unpack (statusMessage status) - return $ responseBody resp +downloadUrl :: Manager -> Nel.NonEmpty String -> IO L.ByteString +downloadUrl manager urls = do + resp <- tryUrls urls + let + allUrls = intercalate ", " $ Nel.toList urls + status = responseStatus resp + unless (statusIsSuccessful status) + $ error $ "Unable to download any of " ++ allUrls + ++ "\n Last status: " ++ show (statusCode status) + ++ " " ++ BC.unpack (statusMessage status) + return $ responseBody resp + where + tryUrl u = do + req <- parseRequest u + httpLbs req manager + + tryUrls (u Nel.:| []) = do + resp <- tryUrl u + return resp + + tryUrls (u Nel.:|(u':us)) = do + resp <- tryUrl u + let status = responseStatus resp + if (statusIsSuccessful status) + then return resp + else tryUrls $ u' Nel.:| us -------------------------------------------------------------------------------- -- JSON data types and instances for parsing the LTS yaml file From a5c393407473c5c44e5a120ebf6754e811583659 Mon Sep 17 00:00:00 2001 From: palmerlao Date: Thu, 4 Apr 2019 10:55:29 -0700 Subject: [PATCH 7/7] Make Hazel try many URLs --- hazel.bzl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/hazel.bzl b/hazel.bzl index 9a0ead0..206643c 100644 --- a/hazel.bzl +++ b/hazel.bzl @@ -11,22 +11,28 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", load("//tools:ghc.bzl", "get_ghc_workspace", "default_ghc_workspaces") load("//tools:mangling.bzl", "hazel_binary", "hazel_library", "hazel_workspace") +def _mirrors(pkg): + return [package_url_template.format(pkg) for package_url_template in [ + "http://hackage.fpcomplete.com/package/{}.tar.gz", + "http://hackage.haskell.org/package/{}.tar.gz", + ]] + def _cabal_haskell_repository_impl(ctx): ghc_workspace = get_ghc_workspace(ctx.attr.ghc_workspaces, ctx) pkg = "{}-{}".format(ctx.attr.package_name, ctx.attr.package_version) - url = "http://hackage.fpcomplete.com/package/{}.tar.gz".format(pkg) + urls = _mirrors(pkg) # If the SHA is wrong, the error message is very unhelpful: # https://github.com/bazelbuild/bazel/issues/3709 # As a workaround, we compute it manually if it's not set (and then fail # this rule). if not ctx.attr.sha256: - ctx.download(url=url, output="tar") + ctx.download(url=urls, output="tar") res = ctx.execute(["openssl", "sha", "-sha256", "tar"]) fail("Missing expected attribute \"sha256\" for {}; computed {}".format(pkg, res.stdout + res.stderr)) ctx.download_and_extract( - url=url, + url=urls, stripPrefix=ctx.attr.package_name + "-" + ctx.attr.package_version, sha256=ctx.attr.sha256, output="") @@ -216,9 +222,7 @@ def hazel_custom_package_hackage( build_file and build_file_content are mutually exclusive. """ package_id = package_name + "-" + version - url = "http://hackage.fpcomplete.com/package/{}.tar.gz".format( - package_id, - ) + urls = _mirrors(package_id) if not build_file and not build_file_content: build_file = "//third_party/haskell:BUILD.{0}".format(package_name) http_archive( @@ -227,7 +231,7 @@ def hazel_custom_package_hackage( build_file_content = build_file_content, sha256 = sha256, strip_prefix = package_id, - urls = [url], + urls = urls, ) def hazel_custom_package_github(