WIP

src/libostree/ostree-fetcher-curl.c

@@ -304,7 +304,6 @@ check_multi_info (OstreeFetcher *fetcher)
       const char *eff_url;
       gboolean is_file;
       gboolean continued_request = FALSE;
-      gboolean should_retry;
 
       if (msg->msg != CURLMSG_DONE)
         continue;
@@ -331,13 +330,10 @@ check_multi_info (OstreeFetcher *fetcher)
             }
           else
             {
-              /* When it is not a file and the OSTREE_CURL_IO_RETRY
-               * environment variable is set, we want to retry the request.
+              /* When it is not a file, we want to retry the request.
                * We accomplish that by using G_IO_ERROR_TIMED_OUT.
                */
-              should_retry = g_getenv ("OSTREE_CURL_IO_RETRY") != NULL;
-              int g_io_error_code
-                  = (is_file || !should_retry) ? G_IO_ERROR_FAILED : G_IO_ERROR_TIMED_OUT;
+              int g_io_error_code = is_file ? G_IO_ERROR_FAILED : G_IO_ERROR_TIMED_OUT;
               g_task_return_new_error (task, G_IO_ERROR, g_io_error_code,
                                        "While fetching %s: [%u] %s", eff_url, curlres,
                                        curl_easy_strerror (curlres));
@@ -920,14 +916,10 @@ initiate_next_curl_request (FetcherRequest *req, GTask *task)
   g_assert_cmpint (rc, ==, CURLM_OK);
   rc = curl_easy_setopt (req->easy, CURLOPT_CONNECTTIMEOUT, 30L);
   g_assert_cmpint (rc, ==, CURLM_OK);
-  /* We used to set CURLOPT_LOW_SPEED_LIMIT and CURLOPT_LOW_SPEED_TIME
-   * here, but see https://github.com/ostreedev/ostree/issues/878#issuecomment-347228854
-   * basically those options don't play well with HTTP2 at the moment
-   * where we can have lots of outstanding requests.  Further,
-   * we could implement that functionality at a higher level
-   * more consistently too.
-   */
-
+  rc = curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_LIMIT, low_speed_limit);
+  g_assert_cmpint (rc, ==, CURLM_OK);
+  rc = curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_TIME, low_speed_time);
+  g_assert_cmpint (rc, ==, CURLM_OK);
   /* closure bindings -> task */
   rc = curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task);
   g_assert_cmpint (rc, ==, CURLM_OK);

src/libostree/ostree-fetcher-curl.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2016 Colin Walters <[email protected]>
+ *
+ * SPDX-License-Identifier: LGPL-2.0+
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#ifndef __GI_SCANNER__
+
+#include "ostree-fetcher.h"
+
+G_BEGIN_DECLS
+
+typedef enum
+{
+  OPT_CURL_TIMEOUT = 300L,
+  OPT_CURL_CONNECT_TIMEOUT = 30L,
+  OPT_CURL_LOWSPEED_LIMIT = 1000L,
+} OstreeCurlOptConfigFlags;
+
+#endif

src/libostree/ostree-fetcher.h

@@ -99,6 +99,10 @@ void _ostree_fetcher_set_proxy (OstreeFetcher *fetcher, const char *proxy);
 void _ostree_fetcher_set_client_cert (OstreeFetcher *fetcher, const char *cert_path,
                                       const char *key_path);
 
+void _ostree_fetcher_set_low_speed_limit (OstreeFetcher *self, long int *opt_low_speed_limit);
+
+void _ostree_fetcher_set_low_speed_time (OstreeFetcher *self, long int *opt_low_speed_time);
+
 void _ostree_fetcher_set_tls_database (OstreeFetcher *self, const char *tlsdb_path);
 
 void _ostree_fetcher_set_extra_headers (OstreeFetcher *self, GVariant *extra_headers);

src/libostree/ostree-repo-pull-private.h

@@ -54,6 +54,9 @@ typedef struct
 
   GVariant *extra_headers;
   char *append_user_agent;
+  long int *low_speed_limit;
+  long int *low_speed_time;
+
 
   gboolean dry_run;
   gboolean dry_run_emitted_progress;

src/libostree/ostree-repo-pull.c

@@ -61,6 +61,8 @@
  * _ostree_fetcher_should_retry_request(). This is the default value for the
  * `n-network-retries` pull option. */
 #define DEFAULT_N_NETWORK_RETRIES 5
+#define OPT_LOWSPEEDLIMIT_DEFAULT 1000L
+#define OPT_LOWSPEEDTIME_DEFAULT 30L
 
 typedef struct
 {
@@ -2928,6 +2930,7 @@ _ostree_repo_cache_summary (OstreeRepo *self, const char *remote, GBytes *summar
 static OstreeFetcher *
 _ostree_repo_remote_new_fetcher (OstreeRepo *self, const char *remote_name, gboolean gzip,
                                  GVariant *extra_headers, const char *append_user_agent,
+                                 long int *low_speed_limit, long int *low_speed_time,
                                  OstreeFetcherSecurityState *out_state, GError **error)
 {
   OstreeFetcher *fetcher = NULL;
@@ -2995,6 +2998,9 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, const char *remote_name, gboo
       }
   }
 
+  _ostree_fetcher_set_low_speed_limit (fetcher, (low_speed_limit) ? low_speed_limit : OPT_LOWSPEEDLIMIT_DEFAULT);
+  _ostree_fetcher_set_low_speed_time (fetcher, (low_speed_time) ? low_speed_time : OPT_LOWSPEEDTIME_DEFAULT);
+
   {
     g_autofree char *tls_ca_path = NULL;
 
@@ -3221,6 +3227,7 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, GError **e
   g_clear_object (&pull_data->fetcher);
   pull_data->fetcher = _ostree_repo_remote_new_fetcher (
       pull_data->repo, remote_name, FALSE, pull_data->extra_headers, pull_data->append_user_agent,
+      pull_data->low_speed_limit, pull_data->low_speed_time,
       &pull_data->fetcher_security_state, error);
   if (pull_data->fetcher == NULL)
     return FALSE;
@@ -3450,6 +3457,10 @@ all_requested_refs_have_commit (
  *   * `n-network-retries` (`u`): Number of times to retry each download on receiving
  *     a transient network error, such as a socket timeout; default is 5, 0
  *     means return errors without retrying. Since: 2018.6
+ *   * `low-speed-limit-bytes` (`u`): The average transfer speed per second of a transfer
+       during the time set via "low-speed-time-seconds" for libcurl to abort.
+ *   * `low-speed-time-seconds` (`u`): The time in number seconds that the transfer
+       speed should be below the "low-speed-limit-bytes" setting for libcurl to abort.
  *   * `ref-keyring-map` (`a(sss)`): Array of (collection ID, ref name, keyring
  *     remote name) tuples specifying which remote's keyring should be used when
  *     doing GPG verification of each collection-ref. This is useful to prevent a
@@ -3570,6 +3581,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, const char *remote_name_or_base
                               &pull_data->timestamp_check_from_rev);
       (void)g_variant_lookup (options, "max-metadata-size", "t", &pull_data->max_metadata_size);
       (void)g_variant_lookup (options, "append-user-agent", "s", &pull_data->append_user_agent);
+      (void)g_variant_lookup (options, "low-speed-limit-bytes", "u", &pull_data->low_speed_limit);
+      (void)g_variant_lookup (options, "low-speed-time-seconds", "u", &pull_data->low_speed_time);
       opt_n_network_retries_set
           = g_variant_lookup (options, "n-network-retries", "u", &pull_data->n_network_retries);
       opt_ref_keyring_map_set
@@ -4895,6 +4908,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, const char *remote_name_or_base
   g_free (pull_data->remote_refspec_name);
   g_free (pull_data->remote_name);
   g_free (pull_data->append_user_agent);
+  g_free (pull_data->low_speed_limit);
+  g_free (pull_data->low_speed_time);
   g_clear_pointer (&pull_data->signapi_commit_verifiers, g_ptr_array_unref);
   g_clear_pointer (&pull_data->signapi_summary_verifiers, g_ptr_array_unref);
   g_clear_pointer (&pull_data->meta_mirrorlist, g_ptr_array_unref);
@@ -5693,7 +5708,7 @@ find_remotes_cb (GObject *obj, GAsyncResult *async_result, gpointer user_data)
                 goto error;
 
               fetcher = _ostree_repo_remote_new_fetcher (self, result->remote->name, TRUE, NULL,
-                                                         NULL, NULL, &error);
+                                                         NULL, NULL, NULL, NULL, &error);
               if (fetcher == NULL)
                 goto error;
 
@@ -6307,6 +6322,8 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, const char *nam
   g_autoptr (GVariant) extra_headers = NULL;
   g_autoptr (GPtrArray) mirrorlist = NULL;
   const char *append_user_agent = NULL;
+  long int *low_speed_limit = NULL;
+  long int *low_speed_time = NULL;
   guint n_network_retries = DEFAULT_N_NETWORK_RETRIES;
   gboolean summary_sig_not_modified = FALSE;
   g_autofree char *summary_sig_if_none_match = NULL;
@@ -6331,6 +6348,8 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, const char *nam
       (void)g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers);
       (void)g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent);
       (void)g_variant_lookup (options, "n-network-retries", "u", &n_network_retries);
+      (void)g_variant_lookup (options, "low-speed-limit-bytes", "u", &low_speed_limit);
+      (void)g_variant_lookup (options, "low-speed-time-seconds", "u", &low_speed_time);
     }
 
   if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error))
@@ -6343,7 +6362,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, const char *nam
   (void)mainctx; // Used for autocleanup
 
   fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, extra_headers, append_user_agent,
-                                             NULL, error);
+                                             low_speed_limit, low_speed_time, NULL, error);
   if (fetcher == NULL)
     return FALSE;