libzypp  16.1.1
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <list>
15 
16 #include "zypp/base/Logger.h"
17 #include "zypp/ExternalProgram.h"
18 #include "zypp/base/String.h"
19 #include "zypp/base/Gettext.h"
20 #include "zypp/base/Sysconfig.h"
21 #include "zypp/base/Gettext.h"
22 
23 #include "zypp/media/MediaCurl.h"
24 #include "zypp/media/ProxyInfo.h"
27 #include "zypp/media/CurlConfig.h"
28 #include "zypp/thread/Once.h"
29 #include "zypp/Target.h"
30 #include "zypp/ZYppFactory.h"
31 #include "zypp/ZConfig.h"
32 
33 #include <cstdlib>
34 #include <sys/types.h>
35 #include <sys/stat.h>
36 #include <sys/mount.h>
37 #include <errno.h>
38 #include <dirent.h>
39 #include <unistd.h>
40 
41 #define DETECT_DIR_INDEX 0
42 #define CONNECT_TIMEOUT 60
43 #define TRANSFER_TIMEOUT_MAX 60 * 60
44 
45 #define EXPLICITLY_NO_PROXY "_none_"
46 
47 #undef CURLVERSION_AT_LEAST
48 #define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O)
49 
50 using namespace std;
51 using namespace zypp::base;
52 
53 namespace
54 {
55  zypp::thread::OnceFlag g_InitOnceFlag = PTHREAD_ONCE_INIT;
56  zypp::thread::OnceFlag g_FreeOnceFlag = PTHREAD_ONCE_INIT;
57 
58  extern "C" void _do_free_once()
59  {
60  curl_global_cleanup();
61  }
62 
63  extern "C" void globalFreeOnce()
64  {
65  zypp::thread::callOnce(g_FreeOnceFlag, _do_free_once);
66  }
67 
68  extern "C" void _do_init_once()
69  {
70  CURLcode ret = curl_global_init( CURL_GLOBAL_ALL );
71  if ( ret != 0 )
72  {
73  WAR << "curl global init failed" << endl;
74  }
75 
76  //
77  // register at exit handler ?
78  // this may cause trouble, because we can protect it
79  // against ourself only.
80  // if the app sets an atexit handler as well, it will
81  // cause a double free while the second of them runs.
82  //
83  //std::atexit( globalFreeOnce);
84  }
85 
86  inline void globalInitOnce()
87  {
88  zypp::thread::callOnce(g_InitOnceFlag, _do_init_once);
89  }
90 
91  int log_curl(CURL *curl, curl_infotype info,
92  char *ptr, size_t len, void *max_lvl)
93  {
94  std::string pfx(" ");
95  long lvl = 0;
96  switch( info)
97  {
98  case CURLINFO_TEXT: lvl = 1; pfx = "*"; break;
99  case CURLINFO_HEADER_IN: lvl = 2; pfx = "<"; break;
100  case CURLINFO_HEADER_OUT: lvl = 2; pfx = ">"; break;
101  default: break;
102  }
103  if( lvl > 0 && max_lvl != NULL && lvl <= *((long *)max_lvl))
104  {
105  std::string msg(ptr, len);
106  std::list<std::string> lines;
107  std::list<std::string>::const_iterator line;
108  zypp::str::split(msg, std::back_inserter(lines), "\r\n");
109  for(line = lines.begin(); line != lines.end(); ++line)
110  {
111  DBG << pfx << " " << *line << endl;
112  }
113  }
114  return 0;
115  }
116 
117  static size_t
118  log_redirects_curl(
119  void *ptr, size_t size, size_t nmemb, void *stream)
120  {
121  // INT << "got header: " << string((char *)ptr, ((char*)ptr) + size*nmemb) << endl;
122 
123  char * lstart = (char *)ptr, * lend = (char *)ptr;
124  size_t pos = 0;
125  size_t max = size * nmemb;
126  while (pos + 1 < max)
127  {
128  // get line
129  for (lstart = lend; *lend != '\n' && pos < max; ++lend, ++pos);
130 
131  // look for "Location"
132  string line(lstart, lend);
133  if (line.find("Location") != string::npos)
134  {
135  DBG << "redirecting to " << line << endl;
136  return max;
137  }
138 
139  // continue with the next line
140  if (pos + 1 < max)
141  {
142  ++lend;
143  ++pos;
144  }
145  else
146  break;
147  }
148 
149  return max;
150  }
151 }
152 
153 namespace zypp {
154  namespace media {
155 
156  namespace {
157  struct ProgressData
158  {
159  ProgressData(CURL *_curl, const long _timeout, const zypp::Url &_url = zypp::Url(),
160  callback::SendReport<DownloadProgressReport> *_report=NULL)
161  : curl(_curl)
162  , timeout(_timeout)
163  , reached(false)
164  , report(_report)
165  , drate_period(-1)
166  , dload_period(0)
167  , secs(0)
168  , drate_avg(-1)
169  , ltime( time(NULL))
170  , dload( 0)
171  , uload( 0)
172  , url(_url)
173  {}
174  CURL *curl;
175  long timeout;
176  bool reached;
177  callback::SendReport<DownloadProgressReport> *report;
178  // download rate of the last period (cca 1 sec)
179  double drate_period;
180  // bytes downloaded at the start of the last period
181  double dload_period;
182  // seconds from the start of the download
183  long secs;
184  // average download rate
185  double drate_avg;
186  // last time the progress was reported
187  time_t ltime;
188  // bytes downloaded at the moment the progress was last reported
189  double dload;
190  // bytes uploaded at the moment the progress was last reported
191  double uload;
193  };
194 
196 
197  inline void escape( string & str_r,
198  const char char_r, const string & escaped_r ) {
199  for ( string::size_type pos = str_r.find( char_r );
200  pos != string::npos; pos = str_r.find( char_r, pos ) ) {
201  str_r.replace( pos, 1, escaped_r );
202  }
203  }
204 
205  inline string escapedPath( string path_r ) {
206  escape( path_r, ' ', "%20" );
207  return path_r;
208  }
209 
210  inline string unEscape( string text_r ) {
211  char * tmp = curl_unescape( text_r.c_str(), 0 );
212  string ret( tmp );
213  curl_free( tmp );
214  return ret;
215  }
216 
217  }
218 
224 {
225  std::string param(url.getQueryParam("timeout"));
226  if( !param.empty())
227  {
228  long num = str::strtonum<long>(param);
229  if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX)
230  s.setTimeout(num);
231  }
232 
233  if ( ! url.getUsername().empty() )
234  {
235  s.setUsername(url.getUsername());
236  if ( url.getPassword().size() )
237  s.setPassword(url.getPassword());
238  }
239  else
240  {
241  // if there is no username, set anonymous auth
242  if ( ( url.getScheme() == "ftp" || url.getScheme() == "tftp" ) && s.username().empty() )
243  s.setAnonymousAuth();
244  }
245 
246  if ( url.getScheme() == "https" )
247  {
248  s.setVerifyPeerEnabled(false);
249  s.setVerifyHostEnabled(false);
250 
251  std::string verify( url.getQueryParam("ssl_verify"));
252  if( verify.empty() ||
253  verify == "yes")
254  {
255  s.setVerifyPeerEnabled(true);
256  s.setVerifyHostEnabled(true);
257  }
258  else if( verify == "no")
259  {
260  s.setVerifyPeerEnabled(false);
261  s.setVerifyHostEnabled(false);
262  }
263  else
264  {
265  std::vector<std::string> flags;
266  std::vector<std::string>::const_iterator flag;
267  str::split( verify, std::back_inserter(flags), ",");
268  for(flag = flags.begin(); flag != flags.end(); ++flag)
269  {
270  if( *flag == "host")
271  s.setVerifyHostEnabled(true);
272  else if( *flag == "peer")
273  s.setVerifyPeerEnabled(true);
274  else
275  ZYPP_THROW(MediaBadUrlException(url, "Unknown ssl_verify flag"));
276  }
277  }
278  }
279 
280  Pathname ca_path( url.getQueryParam("ssl_capath") );
281  if( ! ca_path.empty())
282  {
283  if( !PathInfo(ca_path).isDir() || ! ca_path.absolute())
284  ZYPP_THROW(MediaBadUrlException(url, "Invalid ssl_capath path"));
285  else
287  }
288 
289  Pathname client_cert( url.getQueryParam("ssl_clientcert") );
290  if( ! client_cert.empty())
291  {
292  if( !PathInfo(client_cert).isFile() || !client_cert.absolute())
293  ZYPP_THROW(MediaBadUrlException(url, "Invalid ssl_clientcert file"));
294  else
295  s.setClientCertificatePath(client_cert);
296  }
297  Pathname client_key( url.getQueryParam("ssl_clientkey") );
298  if( ! client_key.empty())
299  {
300  if( !PathInfo(client_key).isFile() || !client_key.absolute())
301  ZYPP_THROW(MediaBadUrlException(url, "Invalid ssl_clientkey file"));
302  else
303  s.setClientKeyPath(client_key);
304  }
305 
306  param = url.getQueryParam( "proxy" );
307  if ( ! param.empty() )
308  {
309  if ( param == EXPLICITLY_NO_PROXY ) {
310  // Workaround TransferSettings shortcoming: With an
311  // empty proxy string, code will continue to look for
312  // valid proxy settings. So set proxy to some non-empty
313  // string, to indicate it has been explicitly disabled.
315  s.setProxyEnabled(false);
316  }
317  else {
318  string proxyport( url.getQueryParam( "proxyport" ) );
319  if ( ! proxyport.empty() ) {
320  param += ":" + proxyport;
321  }
322  s.setProxy(param);
323  s.setProxyEnabled(true);
324  }
325  }
326 
327  param = url.getQueryParam( "proxyuser" );
328  if ( ! param.empty() )
329  {
330  s.setProxyUsername(param);
331  s.setProxyPassword(url.getQueryParam( "proxypass" ));
332  }
333 
334  // HTTP authentication type
335  param = url.getQueryParam("auth");
336  if (!param.empty() && (url.getScheme() == "http" || url.getScheme() == "https"))
337  {
338  try
339  {
340  CurlAuthData::auth_type_str2long(param); // check if we know it
341  }
342  catch (MediaException & ex_r)
343  {
344  DBG << "Rethrowing as MediaUnauthorizedException.";
345  ZYPP_THROW(MediaUnauthorizedException(url, ex_r.msg(), "", ""));
346  }
347  s.setAuthType(param);
348  }
349 
350  // workarounds
351  param = url.getQueryParam("head_requests");
352  if( !param.empty() && param == "no" )
353  s.setHeadRequestsAllowed(false);
354 }
355 
361 {
362  ProxyInfo proxy_info;
363  if ( proxy_info.useProxyFor( url ) )
364  {
365  // We must extract any 'user:pass' from the proxy url
366  // otherwise they won't make it into curl (.curlrc wins).
367  try {
368  Url u( proxy_info.proxy( url ) );
369  s.setProxy( u.asString( url::ViewOption::WITH_SCHEME + url::ViewOption::WITH_HOST + url::ViewOption::WITH_PORT ) );
370  // don't overwrite explicit auth settings
371  if ( s.proxyUsername().empty() )
372  {
373  s.setProxyUsername( u.getUsername( url::E_ENCODED ) );
374  s.setProxyPassword( u.getPassword( url::E_ENCODED ) );
375  }
376  s.setProxyEnabled( true );
377  }
378  catch (...) {} // no proxy if URL is malformed
379  }
380 }
381 
382 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
383 
388 static const char *const anonymousIdHeader()
389 {
390  // we need to add the release and identifier to the
391  // agent string.
392  // The target could be not initialized, and then this information
393  // is guessed.
394  static const std::string _value(
396  "X-ZYpp-AnonymousId: %s",
397  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str() ) )
398  );
399  return _value.c_str();
400 }
401 
406 static const char *const distributionFlavorHeader()
407 {
408  // we need to add the release and identifier to the
409  // agent string.
410  // The target could be not initialized, and then this information
411  // is guessed.
412  static const std::string _value(
414  "X-ZYpp-DistributionFlavor: %s",
415  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str() ) )
416  );
417  return _value.c_str();
418 }
419 
424 static const char *const agentString()
425 {
426  // we need to add the release and identifier to the
427  // agent string.
428  // The target could be not initialized, and then this information
429  // is guessed.
430  static const std::string _value(
431  str::form(
432  "ZYpp %s (curl %s) %s"
433  , VERSION
434  , curl_version_info(CURLVERSION_NOW)->version
435  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
436  )
437  );
438  return _value.c_str();
439 }
440 
441 // we use this define to unbloat code as this C setting option
442 // and catching exception is done frequently.
444 #define SET_OPTION(opt,val) do { \
445  ret = curl_easy_setopt ( _curl, opt, val ); \
446  if ( ret != 0) { \
447  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
448  } \
449  } while ( false )
450 
451 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
452 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
453 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
454 
455 MediaCurl::MediaCurl( const Url & url_r,
456  const Pathname & attach_point_hint_r )
457  : MediaHandler( url_r, attach_point_hint_r,
458  "/", // urlpath at attachpoint
459  true ), // does_download
460  _curl( NULL ),
461  _customHeaders(0L)
462 {
463  _curlError[0] = '\0';
464  _curlDebug = 0L;
465 
466  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
467 
468  globalInitOnce();
469 
470  if( !attachPoint().empty())
471  {
472  PathInfo ainfo(attachPoint());
473  Pathname apath(attachPoint() + "XXXXXX");
474  char *atemp = ::strdup( apath.asString().c_str());
475  char *atest = NULL;
476  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
477  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
478  {
479  WAR << "attach point " << ainfo.path()
480  << " is not useable for " << url_r.getScheme() << endl;
481  setAttachPoint("", true);
482  }
483  else if( atest != NULL)
484  ::rmdir(atest);
485 
486  if( atemp != NULL)
487  ::free(atemp);
488  }
489 }
490 
492 {
493  Url curlUrl (url);
494  curlUrl.setUsername( "" );
495  curlUrl.setPassword( "" );
496  curlUrl.setPathParams( "" );
497  curlUrl.setFragment( "" );
498  curlUrl.delQueryParam("cookies");
499  curlUrl.delQueryParam("proxy");
500  curlUrl.delQueryParam("proxyport");
501  curlUrl.delQueryParam("proxyuser");
502  curlUrl.delQueryParam("proxypass");
503  curlUrl.delQueryParam("ssl_capath");
504  curlUrl.delQueryParam("ssl_verify");
505  curlUrl.delQueryParam("ssl_clientcert");
506  curlUrl.delQueryParam("timeout");
507  curlUrl.delQueryParam("auth");
508  curlUrl.delQueryParam("username");
509  curlUrl.delQueryParam("password");
510  curlUrl.delQueryParam("mediahandler");
511  curlUrl.delQueryParam("credentials");
512  curlUrl.delQueryParam("head_requests");
513  return curlUrl;
514 }
515 
517 {
518  return _settings;
519 }
520 
521 
522 void MediaCurl::setCookieFile( const Pathname &fileName )
523 {
524  _cookieFile = fileName;
525 }
526 
528 
529 void MediaCurl::checkProtocol(const Url &url) const
530 {
531  curl_version_info_data *curl_info = NULL;
532  curl_info = curl_version_info(CURLVERSION_NOW);
533  // curl_info does not need any free (is static)
534  if (curl_info->protocols)
535  {
536  const char * const *proto;
537  std::string scheme( url.getScheme());
538  bool found = false;
539  for(proto=curl_info->protocols; !found && *proto; ++proto)
540  {
541  if( scheme == std::string((const char *)*proto))
542  found = true;
543  }
544  if( !found)
545  {
546  std::string msg("Unsupported protocol '");
547  msg += scheme;
548  msg += "'";
550  }
551  }
552 }
553 
555 {
556  {
557  char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
558  _curlDebug = (ptr && *ptr) ? str::strtonum<long>( ptr) : 0L;
559  if( _curlDebug > 0)
560  {
561  curl_easy_setopt( _curl, CURLOPT_VERBOSE, 1L);
562  curl_easy_setopt( _curl, CURLOPT_DEBUGFUNCTION, log_curl);
563  curl_easy_setopt( _curl, CURLOPT_DEBUGDATA, &_curlDebug);
564  }
565  }
566 
567  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
568  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
569  if ( ret != 0 ) {
570  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
571  }
572 
573  SET_OPTION(CURLOPT_FAILONERROR, 1L);
574  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
575 
576  // create non persistant settings
577  // so that we don't add headers twice
578  TransferSettings vol_settings(_settings);
579 
580  // add custom headers for download.opensuse.org (bsc#955801)
581  if ( _url.getHost() == "download.opensuse.org" )
582  {
583  vol_settings.addHeader(anonymousIdHeader());
584  vol_settings.addHeader(distributionFlavorHeader());
585  }
586  vol_settings.addHeader("Pragma:");
587 
588  _settings.setTimeout(ZConfig::instance().download_transfer_timeout());
590 
592 
593  // fill some settings from url query parameters
594  try
595  {
597  }
598  catch ( const MediaException &e )
599  {
600  disconnectFrom();
601  ZYPP_RETHROW(e);
602  }
603  // if the proxy was not set (or explicitly unset) by url, then look...
604  if ( _settings.proxy().empty() )
605  {
606  // ...at the system proxy settings
608  }
609 
613  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
614  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
615  // just in case curl does not trigger its progress callback frequently
616  // enough.
617  if ( _settings.timeout() )
618  {
619  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
620  }
621 
622  // follow any Location: header that the server sends as part of
623  // an HTTP header (#113275)
624  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
625  // 3 redirects seem to be too few in some cases (bnc #465532)
626  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
627 
628  if ( _url.getScheme() == "https" )
629  {
630 #if CURLVERSION_AT_LEAST(7,19,4)
631  // restrict following of redirections from https to https only
632  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
633 #endif
634 
637  {
638  SET_OPTION(CURLOPT_CAPATH, _settings.certificateAuthoritiesPath().c_str());
639  }
640 
641  if( ! _settings.clientCertificatePath().empty() )
642  {
643  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
644  }
645  if( ! _settings.clientKeyPath().empty() )
646  {
647  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
648  }
649 
650 #ifdef CURLSSLOPT_ALLOW_BEAST
651  // see bnc#779177
652  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
653  if ( ret != 0 ) {
654  disconnectFrom();
656  }
657 #endif
658  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
659  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
660  // bnc#903405 - POODLE: libzypp should only talk TLS
661  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
662  }
663 
664  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
665 
666  /*---------------------------------------------------------------*
667  CURLOPT_USERPWD: [user name]:[password]
668 
669  Url::username/password -> CURLOPT_USERPWD
670  If not provided, anonymous FTP identification
671  *---------------------------------------------------------------*/
672 
673  if ( _settings.userPassword().size() )
674  {
675  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
676  string use_auth = _settings.authType();
677  if (use_auth.empty())
678  use_auth = "digest,basic"; // our default
679  long auth = CurlAuthData::auth_type_str2long(use_auth);
680  if( auth != CURLAUTH_NONE)
681  {
682  DBG << "Enabling HTTP authentication methods: " << use_auth
683  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
684  SET_OPTION(CURLOPT_HTTPAUTH, auth);
685  }
686  }
687 
688  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
689  {
690  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
691  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
692  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
693  /*---------------------------------------------------------------*
694  * CURLOPT_PROXYUSERPWD: [user name]:[password]
695  *
696  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
697  * If not provided, $HOME/.curlrc is evaluated
698  *---------------------------------------------------------------*/
699 
700  string proxyuserpwd = _settings.proxyUserPassword();
701 
702  if ( proxyuserpwd.empty() )
703  {
704  CurlConfig curlconf;
705  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
706  if ( curlconf.proxyuserpwd.empty() )
707  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
708  else
709  {
710  proxyuserpwd = curlconf.proxyuserpwd;
711  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
712  }
713  }
714  else
715  {
716  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
717  }
718 
719  if ( ! proxyuserpwd.empty() )
720  {
721  SET_OPTION(CURLOPT_PROXYUSERPWD, unEscape( proxyuserpwd ).c_str());
722  }
723  }
724 #if CURLVERSION_AT_LEAST(7,19,4)
725  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
726  {
727  // Explicitly disabled in URL (see fillSettingsFromUrl()).
728  // This should also prevent libcurl from looking into the environment.
729  DBG << "Proxy: explicitly NOPROXY" << endl;
730  SET_OPTION(CURLOPT_NOPROXY, "*");
731  }
732 #endif
733  else
734  {
735  DBG << "Proxy: not explicitly set" << endl;
736  DBG << "Proxy: libcurl may look into the environment" << endl;
737  }
738 
740  if ( _settings.minDownloadSpeed() != 0 )
741  {
742  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
743  // default to 10 seconds at low speed
744  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
745  }
746 
747 #if CURLVERSION_AT_LEAST(7,15,5)
748  if ( _settings.maxDownloadSpeed() != 0 )
749  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
750 #endif
751 
752  /*---------------------------------------------------------------*
753  *---------------------------------------------------------------*/
754 
755  _currentCookieFile = _cookieFile.asString();
756  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
757  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
758  else
759  MIL << "No cookies requested" << endl;
760  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
761  SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
762  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
763 
764 #if CURLVERSION_AT_LEAST(7,18,0)
765  // bnc #306272
766  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
767 #endif
768  // append settings custom headers to curl
769  for ( TransferSettings::Headers::const_iterator it = vol_settings.headersBegin();
770  it != vol_settings.headersEnd();
771  ++it )
772  {
773  // MIL << "HEADER " << *it << std::endl;
774 
775  _customHeaders = curl_slist_append(_customHeaders, it->c_str());
776  if ( !_customHeaders )
778  }
779 
780  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
781 }
782 
784 
785 
786 void MediaCurl::attachTo (bool next)
787 {
788  if ( next )
790 
791  if ( !_url.isValid() )
793 
796  {
797  std::string mountpoint = createAttachPoint().asString();
798 
799  if( mountpoint.empty())
801 
802  setAttachPoint( mountpoint, true);
803  }
804 
805  disconnectFrom(); // clean _curl if needed
806  _curl = curl_easy_init();
807  if ( !_curl ) {
809  }
810  try
811  {
812  setupEasy();
813  }
814  catch (Exception & ex)
815  {
816  disconnectFrom();
817  ZYPP_RETHROW(ex);
818  }
819 
820  // FIXME: need a derived class to propelly compare url's
822  setMediaSource(media);
823 }
824 
825 bool
826 MediaCurl::checkAttachPoint(const Pathname &apoint) const
827 {
828  return MediaHandler::checkAttachPoint( apoint, true, true);
829 }
830 
832 
834 {
835  if ( _customHeaders )
836  {
837  curl_slist_free_all(_customHeaders);
838  _customHeaders = 0L;
839  }
840 
841  if ( _curl )
842  {
843  curl_easy_cleanup( _curl );
844  _curl = NULL;
845  }
846 }
847 
849 
850 void MediaCurl::releaseFrom( const std::string & ejectDev )
851 {
852  disconnect();
853 }
854 
855 Url MediaCurl::getFileUrl( const Pathname & filename_r ) const
856 {
857  // Simply extend the URLs pathname. An 'absolute' URL path
858  // is achieved by encoding the leading '/' in an URL path:
859  // URL: ftp://user@server -> ~user
860  // URL: ftp://user@server/ -> ~user
861  // URL: ftp://user@server// -> ~user
862  // URL: ftp://user@server/%2F -> /
863  // ^- this '/' is just a separator
864  Url newurl( _url );
865  newurl.setPathName( ( Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) );
866  return newurl;
867 }
868 
870 
871 void MediaCurl::getFile( const Pathname & filename ) const
872 {
873  // Use absolute file name to prevent access of files outside of the
874  // hierarchy below the attach point.
875  getFileCopy(filename, localPath(filename).absolutename());
876 }
877 
879 
880 void MediaCurl::getFileCopy( const Pathname & filename , const Pathname & target) const
881 {
883 
884  Url fileurl(getFileUrl(filename));
885 
886  bool retry = false;
887 
888  do
889  {
890  try
891  {
892  doGetFileCopy(filename, target, report);
893  retry = false;
894  }
895  // retry with proper authentication data
896  catch (MediaUnauthorizedException & ex_r)
897  {
898  if(authenticate(ex_r.hint(), !retry))
899  retry = true;
900  else
901  {
902  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
903  ZYPP_RETHROW(ex_r);
904  }
905  }
906  // unexpected exception
907  catch (MediaException & excpt_r)
908  {
909  // FIXME: error number fix
910  report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory());
911  ZYPP_RETHROW(excpt_r);
912  }
913  }
914  while (retry);
915 
916  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
917 }
918 
920 
921 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
922 {
923  bool retry = false;
924 
925  do
926  {
927  try
928  {
929  return doGetDoesFileExist( filename );
930  }
931  // authentication problem, retry with proper authentication data
932  catch (MediaUnauthorizedException & ex_r)
933  {
934  if(authenticate(ex_r.hint(), !retry))
935  retry = true;
936  else
937  ZYPP_RETHROW(ex_r);
938  }
939  // unexpected exception
940  catch (MediaException & excpt_r)
941  {
942  ZYPP_RETHROW(excpt_r);
943  }
944  }
945  while (retry);
946 
947  return false;
948 }
949 
951 
952 void MediaCurl::evaluateCurlCode( const Pathname &filename,
953  CURLcode code,
954  bool timeout_reached ) const
955 {
956  if ( code != 0 )
957  {
958  Url url;
959  if (filename.empty())
960  url = _url;
961  else
962  url = getFileUrl(filename);
963  std::string err;
964  try
965  {
966  switch ( code )
967  {
968  case CURLE_UNSUPPORTED_PROTOCOL:
969  case CURLE_URL_MALFORMAT:
970  case CURLE_URL_MALFORMAT_USER:
971  err = " Bad URL";
972  break;
973  case CURLE_LOGIN_DENIED:
974  ZYPP_THROW(
975  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
976  break;
977  case CURLE_HTTP_RETURNED_ERROR:
978  {
979  long httpReturnCode = 0;
980  CURLcode infoRet = curl_easy_getinfo( _curl,
981  CURLINFO_RESPONSE_CODE,
982  &httpReturnCode );
983  if ( infoRet == CURLE_OK )
984  {
985  string msg = "HTTP response: " + str::numstring( httpReturnCode );
986  switch ( httpReturnCode )
987  {
988  case 401:
989  {
990  string auth_hint = getAuthHint();
991 
992  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
993  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
994 
996  url, "Login failed.", _curlError, auth_hint
997  ));
998  }
999 
1000  case 503: // service temporarily unavailable (bnc #462545)
1002  case 504: // gateway timeout
1004  case 403:
1005  {
1006  string msg403;
1007  if (url.asString().find("novell.com") != string::npos)
1008  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
1009  ZYPP_THROW(MediaForbiddenException(url, msg403));
1010  }
1011  case 404:
1013  }
1014 
1015  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
1017  }
1018  else
1019  {
1020  string msg = "Unable to retrieve HTTP response:";
1021  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
1023  }
1024  }
1025  break;
1026  case CURLE_FTP_COULDNT_RETR_FILE:
1027 #if CURLVERSION_AT_LEAST(7,16,0)
1028  case CURLE_REMOTE_FILE_NOT_FOUND:
1029 #endif
1030  case CURLE_FTP_ACCESS_DENIED:
1031  case CURLE_TFTP_NOTFOUND:
1032  err = "File not found";
1034  break;
1035  case CURLE_BAD_PASSWORD_ENTERED:
1036  case CURLE_FTP_USER_PASSWORD_INCORRECT:
1037  err = "Login failed";
1038  break;
1039  case CURLE_COULDNT_RESOLVE_PROXY:
1040  case CURLE_COULDNT_RESOLVE_HOST:
1041  case CURLE_COULDNT_CONNECT:
1042  case CURLE_FTP_CANT_GET_HOST:
1043  err = "Connection failed";
1044  break;
1045  case CURLE_WRITE_ERROR:
1046  err = "Write error";
1047  break;
1048  case CURLE_PARTIAL_FILE:
1049  case CURLE_OPERATION_TIMEDOUT:
1050  timeout_reached = true; // fall though to TimeoutException
1051  // fall though...
1052  case CURLE_ABORTED_BY_CALLBACK:
1053  if( timeout_reached )
1054  {
1055  err = "Timeout reached";
1057  }
1058  else
1059  {
1060  err = "User abort";
1061  }
1062  break;
1063  case CURLE_SSL_PEER_CERTIFICATE:
1064  default:
1065  err = "Unrecognized error";
1066  break;
1067  }
1068 
1069  // uhm, no 0 code but unknown curl exception
1071  }
1072  catch (const MediaException & excpt_r)
1073  {
1074  ZYPP_RETHROW(excpt_r);
1075  }
1076  }
1077  else
1078  {
1079  // actually the code is 0, nothing happened
1080  }
1081 }
1082 
1084 
1085 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
1086 {
1087  DBG << filename.asString() << endl;
1088 
1089  if(!_url.isValid())
1091 
1092  if(_url.getHost().empty())
1094 
1095  Url url(getFileUrl(filename));
1096 
1097  DBG << "URL: " << url.asString() << endl;
1098  // Use URL without options and without username and passwd
1099  // (some proxies dislike them in the URL).
1100  // Curl seems to need the just scheme, hostname and a path;
1101  // the rest was already passed as curl options (in attachTo).
1102  Url curlUrl( clearQueryString(url) );
1103 
1104  //
1105  // See also Bug #154197 and ftp url definition in RFC 1738:
1106  // The url "ftp://user@host/foo/bar/file" contains a path,
1107  // that is relative to the user's home.
1108  // The url "ftp://user@host//foo/bar/file" (or also with
1109  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1110  // contains an absolute path.
1111  //
1112  string urlBuffer( curlUrl.asString());
1113  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1114  urlBuffer.c_str() );
1115  if ( ret != 0 ) {
1117  }
1118 
1119  // instead of returning no data with NOBODY, we return
1120  // little data, that works with broken servers, and
1121  // works for ftp as well, because retrieving only headers
1122  // ftp will return always OK code ?
1123  // See http://curl.haxx.se/docs/knownbugs.html #58
1124  if ( (_url.getScheme() == "http" || _url.getScheme() == "https") &&
1126  ret = curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1127  else
1128  ret = curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1129 
1130  if ( ret != 0 ) {
1131  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1132  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1133  /* yes, this is why we never got to get NOBODY working before,
1134  because setting it changes this option too, and we also
1135  need to reset it
1136  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1137  */
1138  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1140  }
1141 
1142  FILE *file = ::fopen( "/dev/null", "w" );
1143  if ( !file ) {
1144  ERR << "fopen failed for /dev/null" << endl;
1145  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1146  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1147  /* yes, this is why we never got to get NOBODY working before,
1148  because setting it changes this option too, and we also
1149  need to reset it
1150  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1151  */
1152  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1153  if ( ret != 0 ) {
1155  }
1156  ZYPP_THROW(MediaWriteException("/dev/null"));
1157  }
1158 
1159  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1160  if ( ret != 0 ) {
1161  ::fclose(file);
1162  std::string err( _curlError);
1163  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1164  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1165  /* yes, this is why we never got to get NOBODY working before,
1166  because setting it changes this option too, and we also
1167  need to reset it
1168  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1169  */
1170  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1171  if ( ret != 0 ) {
1173  }
1175  }
1176 
1177  CURLcode ok = curl_easy_perform( _curl );
1178  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1179 
1180  // reset curl settings
1181  if ( _url.getScheme() == "http" || _url.getScheme() == "https" )
1182  {
1183  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1184  if ( ret != 0 ) {
1186  }
1187 
1188  /* yes, this is why we never got to get NOBODY working before,
1189  because setting it changes this option too, and we also
1190  need to reset it
1191  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1192  */
1193  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L);
1194  if ( ret != 0 ) {
1196  }
1197 
1198  }
1199  else
1200  {
1201  // for FTP we set different options
1202  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL);
1203  if ( ret != 0 ) {
1205  }
1206  }
1207 
1208  // if the code is not zero, close the file
1209  if ( ok != 0 )
1210  ::fclose(file);
1211 
1212  // as we are not having user interaction, the user can't cancel
1213  // the file existence checking, a callback or timeout return code
1214  // will be always a timeout.
1215  try {
1216  evaluateCurlCode( filename, ok, true /* timeout */);
1217  }
1218  catch ( const MediaFileNotFoundException &e ) {
1219  // if the file did not exist then we can return false
1220  return false;
1221  }
1222  catch ( const MediaException &e ) {
1223  // some error, we are not sure about file existence, rethrw
1224  ZYPP_RETHROW(e);
1225  }
1226  // exists
1227  return ( ok == CURLE_OK );
1228 }
1229 
1231 
1232 
1233 #if DETECT_DIR_INDEX
1234 bool MediaCurl::detectDirIndex() const
1235 {
1236  if(_url.getScheme() != "http" && _url.getScheme() != "https")
1237  return false;
1238  //
1239  // try to check the effective url and set the not_a_file flag
1240  // if the url path ends with a "/", what usually means, that
1241  // we've received a directory index (index.html content).
1242  //
1243  // Note: This may be dangerous and break file retrieving in
1244  // case of some server redirections ... ?
1245  //
1246  bool not_a_file = false;
1247  char *ptr = NULL;
1248  CURLcode ret = curl_easy_getinfo( _curl,
1249  CURLINFO_EFFECTIVE_URL,
1250  &ptr);
1251  if ( ret == CURLE_OK && ptr != NULL)
1252  {
1253  try
1254  {
1255  Url eurl( ptr);
1256  std::string path( eurl.getPathName());
1257  if( !path.empty() && path != "/" && *path.rbegin() == '/')
1258  {
1259  DBG << "Effective url ("
1260  << eurl
1261  << ") seems to provide the index of a directory"
1262  << endl;
1263  not_a_file = true;
1264  }
1265  }
1266  catch( ... )
1267  {}
1268  }
1269  return not_a_file;
1270 }
1271 #endif
1272 
1274 
1275 void MediaCurl::doGetFileCopy( const Pathname & filename , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1276 {
1277  Pathname dest = target.absolutename();
1278  if( assert_dir( dest.dirname() ) )
1279  {
1280  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1281  Url url(getFileUrl(filename));
1282  ZYPP_THROW( MediaSystemException(url, "System error on " + dest.dirname().asString()) );
1283  }
1284  string destNew = target.asString() + ".new.zypp.XXXXXX";
1285  char *buf = ::strdup( destNew.c_str());
1286  if( !buf)
1287  {
1288  ERR << "out of memory for temp file name" << endl;
1289  Url url(getFileUrl(filename));
1290  ZYPP_THROW(MediaSystemException(url, "out of memory for temp file name"));
1291  }
1292 
1293  int tmp_fd = ::mkostemp( buf, O_CLOEXEC );
1294  if( tmp_fd == -1)
1295  {
1296  free( buf);
1297  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1298  ZYPP_THROW(MediaWriteException(destNew));
1299  }
1300  destNew = buf;
1301  free( buf);
1302 
1303  FILE *file = ::fdopen( tmp_fd, "we" );
1304  if ( !file ) {
1305  ::close( tmp_fd);
1306  filesystem::unlink( destNew );
1307  ERR << "fopen failed for file '" << destNew << "'" << endl;
1308  ZYPP_THROW(MediaWriteException(destNew));
1309  }
1310 
1311  DBG << "dest: " << dest << endl;
1312  DBG << "temp: " << destNew << endl;
1313 
1314  // set IFMODSINCE time condition (no download if not modified)
1315  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1316  {
1317  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1318  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1319  }
1320  else
1321  {
1322  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1323  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1324  }
1325  try
1326  {
1327  doGetFileCopyFile(filename, dest, file, report, options);
1328  }
1329  catch (Exception &e)
1330  {
1331  ::fclose( file );
1332  filesystem::unlink( destNew );
1333  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1334  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1335  ZYPP_RETHROW(e);
1336  }
1337 
1338  long httpReturnCode = 0;
1339  CURLcode infoRet = curl_easy_getinfo(_curl,
1340  CURLINFO_RESPONSE_CODE,
1341  &httpReturnCode);
1342  bool modified = true;
1343  if (infoRet == CURLE_OK)
1344  {
1345  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1346  if ( httpReturnCode == 304
1347  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1348  {
1349  DBG << " Not modified.";
1350  modified = false;
1351  }
1352  DBG << endl;
1353  }
1354  else
1355  {
1356  WAR << "Could not get the reponse code." << endl;
1357  }
1358 
1359  if (modified || infoRet != CURLE_OK)
1360  {
1361  // apply umask
1362  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1363  {
1364  ERR << "Failed to chmod file " << destNew << endl;
1365  }
1366  if (::fclose( file ))
1367  {
1368  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1369  ZYPP_THROW(MediaWriteException(destNew));
1370  }
1371  // move the temp file into dest
1372  if ( rename( destNew, dest ) != 0 ) {
1373  ERR << "Rename failed" << endl;
1375  }
1376  }
1377  else
1378  {
1379  // close and remove the temp file
1380  ::fclose( file );
1381  filesystem::unlink( destNew );
1382  }
1383 
1384  DBG << "done: " << PathInfo(dest) << endl;
1385 }
1386 
1388 
1389 void MediaCurl::doGetFileCopyFile( const Pathname & filename , const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1390 {
1391  DBG << filename.asString() << endl;
1392 
1393  if(!_url.isValid())
1395 
1396  if(_url.getHost().empty())
1398 
1399  Url url(getFileUrl(filename));
1400 
1401  DBG << "URL: " << url.asString() << endl;
1402  // Use URL without options and without username and passwd
1403  // (some proxies dislike them in the URL).
1404  // Curl seems to need the just scheme, hostname and a path;
1405  // the rest was already passed as curl options (in attachTo).
1406  Url curlUrl( clearQueryString(url) );
1407 
1408  //
1409  // See also Bug #154197 and ftp url definition in RFC 1738:
1410  // The url "ftp://user@host/foo/bar/file" contains a path,
1411  // that is relative to the user's home.
1412  // The url "ftp://user@host//foo/bar/file" (or also with
1413  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1414  // contains an absolute path.
1415  //
1416  string urlBuffer( curlUrl.asString());
1417  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1418  urlBuffer.c_str() );
1419  if ( ret != 0 ) {
1421  }
1422 
1423  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1424  if ( ret != 0 ) {
1426  }
1427 
1428  // Set callback and perform.
1429  ProgressData progressData(_curl, _settings.timeout(), url, &report);
1430  if (!(options & OPTION_NO_REPORT_START))
1431  report->start(url, dest);
1432  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1433  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1434  }
1435 
1436  ret = curl_easy_perform( _curl );
1437 #if CURLVERSION_AT_LEAST(7,19,4)
1438  // bnc#692260: If the client sends a request with an If-Modified-Since header
1439  // with a future date for the server, the server may respond 200 sending a
1440  // zero size file.
1441  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1442  if ( ftell(file) == 0 && ret == 0 )
1443  {
1444  long httpReturnCode = 33;
1445  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1446  {
1447  long conditionUnmet = 33;
1448  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1449  {
1450  WAR << "TIMECONDITION unmet - retry without." << endl;
1451  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1452  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1453  ret = curl_easy_perform( _curl );
1454  }
1455  }
1456  }
1457 #endif
1458 
1459  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1460  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1461  }
1462 
1463  if ( ret != 0 )
1464  {
1465  ERR << "curl error: " << ret << ": " << _curlError
1466  << ", temp file size " << ftell(file)
1467  << " bytes." << endl;
1468 
1469  // the timeout is determined by the progress data object
1470  // which holds whether the timeout was reached or not,
1471  // otherwise it would be a user cancel
1472  try {
1473  evaluateCurlCode( filename, ret, progressData.reached);
1474  }
1475  catch ( const MediaException &e ) {
1476  // some error, we are not sure about file existence, rethrw
1477  ZYPP_RETHROW(e);
1478  }
1479  }
1480 
1481 #if DETECT_DIR_INDEX
1482  if (!ret && detectDirIndex())
1483  {
1485  }
1486 #endif // DETECT_DIR_INDEX
1487 }
1488 
1490 
1491 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1492 {
1493  filesystem::DirContent content;
1494  getDirInfo( content, dirname, /*dots*/false );
1495 
1496  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1497  Pathname filename = dirname + it->name;
1498  int res = 0;
1499 
1500  switch ( it->type ) {
1501  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1502  case filesystem::FT_FILE:
1503  getFile( filename );
1504  break;
1505  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1506  if ( recurse_r ) {
1507  getDir( filename, recurse_r );
1508  } else {
1509  res = assert_dir( localPath( filename ) );
1510  if ( res ) {
1511  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1512  }
1513  }
1514  break;
1515  default:
1516  // don't provide devices, sockets, etc.
1517  break;
1518  }
1519  }
1520 }
1521 
1523 
1524 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1525  const Pathname & dirname, bool dots ) const
1526 {
1527  getDirectoryYast( retlist, dirname, dots );
1528 }
1529 
1531 
1533  const Pathname & dirname, bool dots ) const
1534 {
1535  getDirectoryYast( retlist, dirname, dots );
1536 }
1537 
1539 
1540 int MediaCurl::progressCallback( void *clientp,
1541  double dltotal, double dlnow,
1542  double ultotal, double ulnow)
1543 {
1544  ProgressData *pdata = reinterpret_cast<ProgressData *>(clientp);
1545  if( pdata)
1546  {
1547  // work around curl bug that gives us old data
1548  long httpReturnCode = 0;
1549  if (curl_easy_getinfo(pdata->curl, CURLINFO_RESPONSE_CODE, &httpReturnCode) != CURLE_OK || httpReturnCode == 0)
1550  return 0;
1551 
1552  time_t now = time(NULL);
1553  if( now > 0)
1554  {
1555  // reset time of last change in case initial time()
1556  // failed or the time was adjusted (goes backward)
1557  if( pdata->ltime <= 0 || pdata->ltime > now)
1558  {
1559  pdata->ltime = now;
1560  }
1561 
1562  // start time counting as soon as first data arrives
1563  // (skip the connection / redirection time at begin)
1564  time_t dif = 0;
1565  if (dlnow > 0 || ulnow > 0)
1566  {
1567  dif = (now - pdata->ltime);
1568  dif = dif > 0 ? dif : 0;
1569 
1570  pdata->secs += dif;
1571  }
1572 
1573  // update the drate_avg and drate_period only after a second has passed
1574  // (this callback is called much more often than a second)
1575  // otherwise the values would be far from accurate when measuring
1576  // the time in seconds
1578 
1579  if ( pdata->secs > 1 && (dif > 0 || dlnow == dltotal ))
1580  pdata->drate_avg = (dlnow / pdata->secs);
1581 
1582  if ( dif > 0 )
1583  {
1584  pdata->drate_period = ((dlnow - pdata->dload_period) / dif);
1585  pdata->dload_period = dlnow;
1586  }
1587  }
1588 
1589  // send progress report first, abort transfer if requested
1590  if( pdata->report)
1591  {
1592  if (!(*(pdata->report))->progress(int( dltotal ? dlnow * 100 / dltotal : 0 ),
1593  pdata->url,
1594  pdata->drate_avg,
1595  pdata->drate_period))
1596  {
1597  return 1; // abort transfer
1598  }
1599  }
1600 
1601  // check if we there is a timeout set
1602  if( pdata->timeout > 0)
1603  {
1604  if( now > 0)
1605  {
1606  bool progress = false;
1607 
1608  // update download data if changed, mark progress
1609  if( dlnow != pdata->dload)
1610  {
1611  progress = true;
1612  pdata->dload = dlnow;
1613  pdata->ltime = now;
1614  }
1615  // update upload data if changed, mark progress
1616  if( ulnow != pdata->uload)
1617  {
1618  progress = true;
1619  pdata->uload = ulnow;
1620  pdata->ltime = now;
1621  }
1622 
1623  if( !progress && (now >= (pdata->ltime + pdata->timeout)))
1624  {
1625  pdata->reached = true;
1626  return 1; // aborts transfer
1627  }
1628  }
1629  }
1630  }
1631  return 0;
1632 }
1633 
1635 {
1636  ProgressData *pdata = reinterpret_cast<ProgressData *>(clientp);
1637  return pdata ? pdata->curl : 0;
1638 }
1639 
1641 
1643 {
1644  long auth_info = CURLAUTH_NONE;
1645 
1646  CURLcode infoRet =
1647  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1648 
1649  if(infoRet == CURLE_OK)
1650  {
1651  return CurlAuthData::auth_type_long2str(auth_info);
1652  }
1653 
1654  return "";
1655 }
1656 
1658 
1659 bool MediaCurl::authenticate(const string & availAuthTypes, bool firstTry) const
1660 {
1662  Target_Ptr target = zypp::getZYpp()->getTarget();
1663  CredentialManager cm(CredManagerOptions(target ? target->root() : ""));
1664  CurlAuthData_Ptr credentials;
1665 
1666  // get stored credentials
1667  AuthData_Ptr cmcred = cm.getCred(_url);
1668 
1669  if (cmcred && firstTry)
1670  {
1671  credentials.reset(new CurlAuthData(*cmcred));
1672  DBG << "got stored credentials:" << endl << *credentials << endl;
1673  }
1674  // if not found, ask user
1675  else
1676  {
1677 
1678  CurlAuthData_Ptr curlcred;
1679  curlcred.reset(new CurlAuthData());
1681 
1682  // preset the username if present in current url
1683  if (!_url.getUsername().empty() && firstTry)
1684  curlcred->setUsername(_url.getUsername());
1685  // if CM has found some credentials, preset the username from there
1686  else if (cmcred)
1687  curlcred->setUsername(cmcred->username());
1688 
1689  // indicate we have no good credentials from CM
1690  cmcred.reset();
1691 
1692  string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1693 
1694  // set available authentication types from the exception
1695  // might be needed in prompt
1696  curlcred->setAuthType(availAuthTypes);
1697 
1698  // ask user
1699  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1700  {
1701  DBG << "callback answer: retry" << endl
1702  << "CurlAuthData: " << *curlcred << endl;
1703 
1704  if (curlcred->valid())
1705  {
1706  credentials = curlcred;
1707  // if (credentials->username() != _url.getUsername())
1708  // _url.setUsername(credentials->username());
1716  }
1717  }
1718  else
1719  {
1720  DBG << "callback answer: cancel" << endl;
1721  }
1722  }
1723 
1724  // set username and password
1725  if (credentials)
1726  {
1727  // HACK, why is this const?
1728  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1729  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1730 
1731  // set username and password
1732  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1734 
1735  // set available authentication types from the exception
1736  if (credentials->authType() == CURLAUTH_NONE)
1737  credentials->setAuthType(availAuthTypes);
1738 
1739  // set auth type (seems this must be set _after_ setting the userpwd)
1740  if (credentials->authType() != CURLAUTH_NONE)
1741  {
1742  // FIXME: only overwrite if not empty?
1743  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1744  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1746  }
1747 
1748  if (!cmcred)
1749  {
1750  credentials->setUrl(_url);
1751  cm.addCred(*credentials);
1752  cm.save();
1753  }
1754 
1755  return true;
1756  }
1757 
1758  return false;
1759 }
1760 
1761 
1762  } // namespace media
1763 } // namespace zypp
1764 //
void setPassword(const std::string &pass, EEncoding eflag=zypp::url::E_DECODED)
Set the password in the URL authority.
Definition: Url.cc:733
std::string userPassword() const
returns the user and password as a user:pass string
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:320
Interface to gettext.
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:529
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:451
#define MIL
Definition: Logger.h:64
#define CONNECT_TIMEOUT
Definition: MediaCurl.cc:42
bool verifyHostEnabled() const
Whether to verify host for ssl.
Pathname clientKeyPath() const
SSL client key file.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:321
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1659
static ZConfig & instance()
Singleton ctor.
Definition: Resolver.cc:121
virtual void releaseFrom(const std::string &ejectDev)
Call concrete handler to release the media.
Definition: MediaCurl.cc:850
const std::string & msg() const
Return the message string provided to the ctor.
Definition: Exception.h:185
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:32
Flag to request encoded string(s).
Definition: UrlUtils.h:53
long connectTimeout() const
connection timeout
Headers::const_iterator headersEnd() const
end iterators to additional headers
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
Definition: Url.cc:598
void setClientKeyPath(const zypp::Pathname &path)
Sets the SSL client key file.
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:44
TransferSettings & settings()
Definition: MediaCurl.cc:516
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:582
Holds transfer setting.
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:491
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
std::string escape(const C_Str &str_r, const char sep_r)
Escape desired character c using a backslash.
Definition: String.cc:369
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Definition: MediaCurl.cc:1540
void setProxyUsername(const std::string &proxyuser)
sets the proxy user
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
Pathname certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
void setPathParams(const std::string &params)
Set the path parameters.
Definition: Url.cc:780
void setHeadRequestsAllowed(bool allowed)
set whether HEAD requests are allowed
Definition: Arch.h:339
pthread_once_t OnceFlag
The OnceFlag variable type.
Definition: Once.h:32
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:566
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
void setConnectTimeout(long t)
set the connect timeout
void setUsername(const std::string &user, EEncoding eflag=zypp::url::E_DECODED)
Set the username in the URL authority.
Definition: Url.cc:724
double dload
Definition: MediaCurl.cc:189
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:554
#define EXPLICITLY_NO_PROXY
Definition: MediaCurl.cc:45
Convenient building of std::string with boost::format.
Definition: String.h:248
Structure holding values of curlrc options.
Definition: CurlConfig.h:16
bool isValid() const
Verifies the Url.
Definition: Url.cc:483
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
Edition * _value
Definition: SysContent.cc:311
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
std::string _currentCookieFile
Definition: MediaCurl.h:167
void setProxy(const std::string &proxyhost)
proxy to use if it is enabled
void setFragment(const std::string &fragment, EEncoding eflag=zypp::url::E_DECODED)
Set the fragment string in the URL.
Definition: Url.cc:716
#define ERR
Definition: Logger.h:66
void setPassword(const std::string &password)
sets the auth password
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:491
void setUsername(const std::string &username)
sets the auth username
bool headRequestsAllowed() const
whether HEAD requests are allowed
void setAnonymousAuth()
sets anonymous authentication (ie: for ftp)
virtual void getFile(const Pathname &filename) const
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:871
std::string proxy(const Url &url) const
Definition: ProxyInfo.cc:44
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:522
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1642
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:329
void setPathName(const std::string &path, EEncoding eflag=zypp::url::E_DECODED)
Set the path name.
Definition: Url.cc:758
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: CurlConfig.cc:24
void doGetFileCopyFile(const Pathname &srcFilename, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1389
std::string userAgentString() const
user agent string
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \t")
Split line_r into words.
Definition: String.h:519
void setProxyPassword(const std::string &proxypass)
sets the proxy password
Abstract base class for &#39;physical&#39; MediaHandler like MediaCD, etc.
Definition: MediaHandler.h:45
void callOnce(OnceFlag &flag, void(*func)())
Call once function.
Definition: Once.h:50
void setAuthType(const std::string &authtype)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:221
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:653
const Url _url
Url to handle.
Definition: MediaHandler.h:110
virtual bool getDoesFileExist(const Pathname &filename) const
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:921
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:667
Just inherits Exception to separate media exceptions.
long timeout
Definition: MediaCurl.cc:175
void disconnect()
Use concrete handler to isconnect media.
do not send a start ProgressReport
Definition: MediaCurl.h:46
#define WAR
Definition: Logger.h:65
TransferSettings _settings
Definition: MediaCurl.h:174
time_t ltime
Definition: MediaCurl.cc:187
bool reached
Definition: MediaCurl.cc:176
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
zypp::Url url
Definition: MediaCurl.cc:192
void setTimeout(long t)
set the transfer timeout
bool useProxyFor(const Url &url_r) const
Return true if enabled and url_r does not match noProxy.
Definition: ProxyInfo.cc:56
#define _(MSG)
Definition: Gettext.h:29
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
static const char *const agentString()
initialized only once, this gets the agent string which also includes the curl version ...
Definition: MediaCurl.cc:424
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
std::string proxyuserpwd
Definition: CurlConfig.h:39
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:654
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:826
shared_ptr< CurlAuthData > CurlAuthData_Ptr
virtual void getDir(const Pathname &dirname, bool recurse_r) const
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1491
std::string numstring(char n, int w=0)
Definition: String.h:305
virtual void disconnectFrom()
Definition: MediaCurl.cc:833
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
SolvableIdType size_type
Definition: PoolMember.h:126
bool detectDirIndex() const
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:36
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
void fillSettingsFromUrl(const Url &url, TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: MediaCurl.cc:223
curl_slist * _customHeaders
Definition: MediaCurl.h:173
Headers::const_iterator headersBegin() const
begin iterators to additional headers
void setClientCertificatePath(const zypp::Pathname &path)
Sets the SSL client certificate file.
shared_ptr< AuthData > AuthData_Ptr
Definition: MediaUserAuth.h:69
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:367
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:444
Pathname attachPoint() const
Return the currently used attach point.
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
Definition: MediaCurl.cc:855
Base class for Exception.
Definition: Exception.h:143
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1524
const std::string & hint() const
comma separated list of available authentication types
static const char *const distributionFlavorHeader()
initialized only once, this gets the distribution flavor from the target, which we pass in the http h...
Definition: MediaCurl.cc:406
void fillSettingsSystemProxy(const Url &url, TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: MediaCurl.cc:360
callback::SendReport< DownloadProgressReport > * report
Definition: MediaCurl.cc:177
void addHeader(const std::string &header)
add a header, on the form "Foo: Bar"
CURL * curl
Definition: MediaCurl.cc:174
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1634
void setCertificateAuthoritiesPath(const zypp::Pathname &path)
Sets the SSL certificate authorities path.
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:445
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:75
virtual void attachTo(bool next=false)
Call concrete handler to attach the media.
Definition: MediaCurl.cc:786
virtual void getFileCopy(const Pathname &srcFilename, const Pathname &targetFilename) const
Definition: MediaCurl.cc:880
double dload_period
Definition: MediaCurl.cc:181
Definition: Fd.cc:28
virtual void doGetFileCopy(const Pathname &srcFilename, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1275
static Pathname _cookieFile
Definition: MediaCurl.h:168
double drate_avg
Definition: MediaCurl.cc:185
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:806
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:1085
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:527
std::string authType() const
get the allowed authentication types
double uload
Definition: MediaCurl.cc:191
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
#define TRANSFER_TIMEOUT_MAX
Definition: MediaCurl.cc:43
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
Curl HTTP authentication data.
Definition: MediaUserAuth.h:74
double drate_period
Definition: MediaCurl.cc:179
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:172
void setVerifyPeerEnabled(bool enabled)
Sets whether to verify host for ssl.
Pathname clientCertificatePath() const
SSL client certificate file.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:952
Url url() const
Url used.
Definition: MediaHandler.h:506
std::string proxy() const
proxy host
bool proxyEnabled() const
proxy is enabled
long secs
Definition: MediaCurl.cc:183
Convenience interface for handling authentication data of media user.
void setVerifyHostEnabled(bool enabled)
Sets whether to verify host for ssl.
Url manipulation class.
Definition: Url.h:87
void setUserAgentString(const std::string &agent)
sets the user agent ie: "Mozilla v3"
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
static const char *const anonymousIdHeader()
initialized only once, this gets the anonymous id from the target, which we pass in the http header ...
Definition: MediaCurl.cc:388
void setProxyEnabled(bool enabled)
whether the proxy is used or not
std::string username() const
auth username
#define DBG
Definition: Logger.h:63
std::string getPassword(EEncoding eflag=zypp::url::E_DECODED) const
Returns the password from the URL authority.
Definition: Url.cc:574
void delQueryParam(const std::string &param)
remove the specified query parameter.
Definition: Url.cc:834
std::string proxyUsername() const
proxy auth username
long timeout() const
transfer timeout