libzypp  15.21.1
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <list>
15 
16 #include "zypp/base/Logger.h"
17 #include "zypp/ExternalProgram.h"
18 #include "zypp/base/String.h"
19 #include "zypp/base/Gettext.h"
20 #include "zypp/base/Sysconfig.h"
21 #include "zypp/base/Gettext.h"
22 
23 #include "zypp/media/MediaCurl.h"
24 #include "zypp/media/ProxyInfo.h"
27 #include "zypp/media/CurlConfig.h"
28 #include "zypp/thread/Once.h"
29 #include "zypp/Target.h"
30 #include "zypp/ZYppFactory.h"
31 #include "zypp/ZConfig.h"
32 
33 #include <cstdlib>
34 #include <sys/types.h>
35 #include <sys/stat.h>
36 #include <sys/mount.h>
37 #include <errno.h>
38 #include <dirent.h>
39 #include <unistd.h>
40 
41 #define DETECT_DIR_INDEX 0
42 #define CONNECT_TIMEOUT 60
43 #define TRANSFER_TIMEOUT_MAX 60 * 60
44 
45 #define EXPLICITLY_NO_PROXY "_none_"
46 
47 #undef CURLVERSION_AT_LEAST
48 #define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O)
49 
50 using namespace std;
51 using namespace zypp::base;
52 
53 namespace
54 {
55  zypp::thread::OnceFlag g_InitOnceFlag = PTHREAD_ONCE_INIT;
56  zypp::thread::OnceFlag g_FreeOnceFlag = PTHREAD_ONCE_INIT;
57 
58  extern "C" void _do_free_once()
59  {
60  curl_global_cleanup();
61  }
62 
63  extern "C" void globalFreeOnce()
64  {
65  zypp::thread::callOnce(g_FreeOnceFlag, _do_free_once);
66  }
67 
68  extern "C" void _do_init_once()
69  {
70  CURLcode ret = curl_global_init( CURL_GLOBAL_ALL );
71  if ( ret != 0 )
72  {
73  WAR << "curl global init failed" << endl;
74  }
75 
76  //
77  // register at exit handler ?
78  // this may cause trouble, because we can protect it
79  // against ourself only.
80  // if the app sets an atexit handler as well, it will
81  // cause a double free while the second of them runs.
82  //
83  //std::atexit( globalFreeOnce);
84  }
85 
86  inline void globalInitOnce()
87  {
88  zypp::thread::callOnce(g_InitOnceFlag, _do_init_once);
89  }
90 
91  int log_curl(CURL *curl, curl_infotype info,
92  char *ptr, size_t len, void *max_lvl)
93  {
94  std::string pfx(" ");
95  long lvl = 0;
96  switch( info)
97  {
98  case CURLINFO_TEXT: lvl = 1; pfx = "*"; break;
99  case CURLINFO_HEADER_IN: lvl = 2; pfx = "<"; break;
100  case CURLINFO_HEADER_OUT: lvl = 2; pfx = ">"; break;
101  default: break;
102  }
103  if( lvl > 0 && max_lvl != NULL && lvl <= *((long *)max_lvl))
104  {
105  std::string msg(ptr, len);
106  std::list<std::string> lines;
107  std::list<std::string>::const_iterator line;
108  zypp::str::split(msg, std::back_inserter(lines), "\r\n");
109  for(line = lines.begin(); line != lines.end(); ++line)
110  {
111  DBG << pfx << " " << *line << endl;
112  }
113  }
114  return 0;
115  }
116 
117  static size_t
118  log_redirects_curl(
119  void *ptr, size_t size, size_t nmemb, void *stream)
120  {
121  // INT << "got header: " << string((char *)ptr, ((char*)ptr) + size*nmemb) << endl;
122 
123  char * lstart = (char *)ptr, * lend = (char *)ptr;
124  size_t pos = 0;
125  size_t max = size * nmemb;
126  while (pos + 1 < max)
127  {
128  // get line
129  for (lstart = lend; *lend != '\n' && pos < max; ++lend, ++pos);
130 
131  // look for "Location"
132  string line(lstart, lend);
133  if (line.find("Location") != string::npos)
134  {
135  DBG << "redirecting to " << line << endl;
136  return max;
137  }
138 
139  // continue with the next line
140  if (pos + 1 < max)
141  {
142  ++lend;
143  ++pos;
144  }
145  else
146  break;
147  }
148 
149  return max;
150  }
151 }
152 
153 namespace zypp {
154  namespace media {
155 
156  namespace {
157  struct ProgressData
158  {
159  ProgressData(CURL *_curl, const long _timeout, const zypp::Url &_url = zypp::Url(),
160  callback::SendReport<DownloadProgressReport> *_report=NULL)
161  : curl(_curl)
162  , timeout(_timeout)
163  , reached(false)
164  , report(_report)
165  , drate_period(-1)
166  , dload_period(0)
167  , secs(0)
168  , drate_avg(-1)
169  , ltime( time(NULL))
170  , dload( 0)
171  , uload( 0)
172  , url(_url)
173  {}
174  CURL *curl;
175  long timeout;
176  bool reached;
177  callback::SendReport<DownloadProgressReport> *report;
178  // download rate of the last period (cca 1 sec)
179  double drate_period;
180  // bytes downloaded at the start of the last period
181  double dload_period;
182  // seconds from the start of the download
183  long secs;
184  // average download rate
185  double drate_avg;
186  // last time the progress was reported
187  time_t ltime;
188  // bytes downloaded at the moment the progress was last reported
189  double dload;
190  // bytes uploaded at the moment the progress was last reported
191  double uload;
193  };
194 
196 
197  inline void escape( string & str_r,
198  const char char_r, const string & escaped_r ) {
199  for ( string::size_type pos = str_r.find( char_r );
200  pos != string::npos; pos = str_r.find( char_r, pos ) ) {
201  str_r.replace( pos, 1, escaped_r );
202  }
203  }
204 
205  inline string escapedPath( string path_r ) {
206  escape( path_r, ' ', "%20" );
207  return path_r;
208  }
209 
210  inline string unEscape( string text_r ) {
211  char * tmp = curl_unescape( text_r.c_str(), 0 );
212  string ret( tmp );
213  curl_free( tmp );
214  return ret;
215  }
216 
217  }
218 
224 {
225  std::string param(url.getQueryParam("timeout"));
226  if( !param.empty())
227  {
228  long num = str::strtonum<long>(param);
229  if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX)
230  s.setTimeout(num);
231  }
232 
233  if ( ! url.getUsername().empty() )
234  {
235  s.setUsername(url.getUsername());
236  if ( url.getPassword().size() )
237  s.setPassword(url.getPassword());
238  }
239  else
240  {
241  // if there is no username, set anonymous auth
242  if ( ( url.getScheme() == "ftp" || url.getScheme() == "tftp" ) && s.username().empty() )
243  s.setAnonymousAuth();
244  }
245 
246  if ( url.getScheme() == "https" )
247  {
248  s.setVerifyPeerEnabled(false);
249  s.setVerifyHostEnabled(false);
250 
251  std::string verify( url.getQueryParam("ssl_verify"));
252  if( verify.empty() ||
253  verify == "yes")
254  {
255  s.setVerifyPeerEnabled(true);
256  s.setVerifyHostEnabled(true);
257  }
258  else if( verify == "no")
259  {
260  s.setVerifyPeerEnabled(false);
261  s.setVerifyHostEnabled(false);
262  }
263  else
264  {
265  std::vector<std::string> flags;
266  std::vector<std::string>::const_iterator flag;
267  str::split( verify, std::back_inserter(flags), ",");
268  for(flag = flags.begin(); flag != flags.end(); ++flag)
269  {
270  if( *flag == "host")
271  s.setVerifyHostEnabled(true);
272  else if( *flag == "peer")
273  s.setVerifyPeerEnabled(true);
274  else
275  ZYPP_THROW(MediaBadUrlException(url, "Unknown ssl_verify flag"));
276  }
277  }
278  }
279 
280  Pathname ca_path( url.getQueryParam("ssl_capath") );
281  if( ! ca_path.empty())
282  {
283  if( !PathInfo(ca_path).isDir() || ! ca_path.absolute())
284  ZYPP_THROW(MediaBadUrlException(url, "Invalid ssl_capath path"));
285  else
287  }
288 
289  Pathname client_cert( url.getQueryParam("ssl_clientcert") );
290  if( ! client_cert.empty())
291  {
292  if( !PathInfo(client_cert).isFile() || !client_cert.absolute())
293  ZYPP_THROW(MediaBadUrlException(url, "Invalid ssl_clientcert file"));
294  else
295  s.setClientCertificatePath(client_cert);
296  }
297  Pathname client_key( url.getQueryParam("ssl_clientkey") );
298  if( ! client_key.empty())
299  {
300  if( !PathInfo(client_key).isFile() || !client_key.absolute())
301  ZYPP_THROW(MediaBadUrlException(url, "Invalid ssl_clientkey file"));
302  else
303  s.setClientKeyPath(client_key);
304  }
305 
306  param = url.getQueryParam( "proxy" );
307  if ( ! param.empty() )
308  {
309  if ( param == EXPLICITLY_NO_PROXY ) {
310  // Workaround TransferSettings shortcoming: With an
311  // empty proxy string, code will continue to look for
312  // valid proxy settings. So set proxy to some non-empty
313  // string, to indicate it has been explicitly disabled.
315  s.setProxyEnabled(false);
316  }
317  else {
318  string proxyport( url.getQueryParam( "proxyport" ) );
319  if ( ! proxyport.empty() ) {
320  param += ":" + proxyport;
321  }
322  s.setProxy(param);
323  s.setProxyEnabled(true);
324  }
325  }
326 
327  param = url.getQueryParam( "proxyuser" );
328  if ( ! param.empty() )
329  {
330  s.setProxyUsername(param);
331  s.setProxyPassword(url.getQueryParam( "proxypass" ));
332  }
333 
334  // HTTP authentication type
335  param = url.getQueryParam("auth");
336  if (!param.empty() && (url.getScheme() == "http" || url.getScheme() == "https"))
337  {
338  try
339  {
340  CurlAuthData::auth_type_str2long(param); // check if we know it
341  }
342  catch (MediaException & ex_r)
343  {
344  DBG << "Rethrowing as MediaUnauthorizedException.";
345  ZYPP_THROW(MediaUnauthorizedException(url, ex_r.msg(), "", ""));
346  }
347  s.setAuthType(param);
348  }
349 
350  // workarounds
351  param = url.getQueryParam("head_requests");
352  if( !param.empty() && param == "no" )
353  s.setHeadRequestsAllowed(false);
354 }
355 
361 {
362  ProxyInfo proxy_info;
363  if ( proxy_info.useProxyFor( url ) )
364  {
365  // We must extract any 'user:pass' from the proxy url
366  // otherwise they won't make it into curl (.curlrc wins).
367  try {
368  Url u( proxy_info.proxy( url ) );
369  s.setProxy( u.asString( url::ViewOption::WITH_SCHEME + url::ViewOption::WITH_HOST + url::ViewOption::WITH_PORT ) );
370  // don't overwrite explicit auth settings
371  if ( s.proxyUsername().empty() )
372  {
373  s.setProxyUsername( u.getUsername( url::E_ENCODED ) );
374  s.setProxyPassword( u.getPassword( url::E_ENCODED ) );
375  }
376  s.setProxyEnabled( true );
377  }
378  catch (...) {} // no proxy if URL is malformed
379  }
380 }
381 
382 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
383 
388 static const char *const anonymousIdHeader()
389 {
390  // we need to add the release and identifier to the
391  // agent string.
392  // The target could be not initialized, and then this information
393  // is guessed.
394  static const std::string _value(
396  "X-ZYpp-AnonymousId: %s",
397  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str() ) )
398  );
399  return _value.c_str();
400 }
401 
406 static const char *const distributionFlavorHeader()
407 {
408  // we need to add the release and identifier to the
409  // agent string.
410  // The target could be not initialized, and then this information
411  // is guessed.
412  static const std::string _value(
414  "X-ZYpp-DistributionFlavor: %s",
415  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str() ) )
416  );
417  return _value.c_str();
418 }
419 
424 static const char *const agentString()
425 {
426  // we need to add the release and identifier to the
427  // agent string.
428  // The target could be not initialized, and then this information
429  // is guessed.
430  static const std::string _value(
431  str::form(
432  "ZYpp %s (curl %s) %s"
433  , VERSION
434  , curl_version_info(CURLVERSION_NOW)->version
435  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
436  )
437  );
438  return _value.c_str();
439 }
440 
441 // we use this define to unbloat code as this C setting option
442 // and catching exception is done frequently.
444 #define SET_OPTION(opt,val) do { \
445  ret = curl_easy_setopt ( _curl, opt, val ); \
446  if ( ret != 0) { \
447  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
448  } \
449  } while ( false )
450 
451 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
452 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
453 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
454 
455 MediaCurl::MediaCurl( const Url & url_r,
456  const Pathname & attach_point_hint_r )
457  : MediaHandler( url_r, attach_point_hint_r,
458  "/", // urlpath at attachpoint
459  true ), // does_download
460  _curl( NULL ),
461  _customHeaders(0L)
462 {
463  _curlError[0] = '\0';
464  _curlDebug = 0L;
465 
466  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
467 
468  globalInitOnce();
469 
470  if( !attachPoint().empty())
471  {
472  PathInfo ainfo(attachPoint());
473  Pathname apath(attachPoint() + "XXXXXX");
474  char *atemp = ::strdup( apath.asString().c_str());
475  char *atest = NULL;
476  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
477  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
478  {
479  WAR << "attach point " << ainfo.path()
480  << " is not useable for " << url_r.getScheme() << endl;
481  setAttachPoint("", true);
482  }
483  else if( atest != NULL)
484  ::rmdir(atest);
485 
486  if( atemp != NULL)
487  ::free(atemp);
488  }
489 }
490 
492 {
493  Url curlUrl (url);
494  curlUrl.setUsername( "" );
495  curlUrl.setPassword( "" );
496  curlUrl.setPathParams( "" );
497  curlUrl.setFragment( "" );
498  curlUrl.delQueryParam("cookies");
499  curlUrl.delQueryParam("proxy");
500  curlUrl.delQueryParam("proxyport");
501  curlUrl.delQueryParam("proxyuser");
502  curlUrl.delQueryParam("proxypass");
503  curlUrl.delQueryParam("ssl_capath");
504  curlUrl.delQueryParam("ssl_verify");
505  curlUrl.delQueryParam("ssl_clientcert");
506  curlUrl.delQueryParam("timeout");
507  curlUrl.delQueryParam("auth");
508  curlUrl.delQueryParam("username");
509  curlUrl.delQueryParam("password");
510  curlUrl.delQueryParam("mediahandler");
511  curlUrl.delQueryParam("credentials");
512  curlUrl.delQueryParam("head_requests");
513  return curlUrl;
514 }
515 
517 {
518  return _settings;
519 }
520 
521 
522 void MediaCurl::setCookieFile( const Pathname &fileName )
523 {
524  _cookieFile = fileName;
525 }
526 
528 
529 void MediaCurl::checkProtocol(const Url &url) const
530 {
531  curl_version_info_data *curl_info = NULL;
532  curl_info = curl_version_info(CURLVERSION_NOW);
533  // curl_info does not need any free (is static)
534  if (curl_info->protocols)
535  {
536  const char * const *proto;
537  std::string scheme( url.getScheme());
538  bool found = false;
539  for(proto=curl_info->protocols; !found && *proto; ++proto)
540  {
541  if( scheme == std::string((const char *)*proto))
542  found = true;
543  }
544  if( !found)
545  {
546  std::string msg("Unsupported protocol '");
547  msg += scheme;
548  msg += "'";
550  }
551  }
552 }
553 
555 {
556  {
557  char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
558  _curlDebug = (ptr && *ptr) ? str::strtonum<long>( ptr) : 0L;
559  if( _curlDebug > 0)
560  {
561  curl_easy_setopt( _curl, CURLOPT_VERBOSE, 1L);
562  curl_easy_setopt( _curl, CURLOPT_DEBUGFUNCTION, log_curl);
563  curl_easy_setopt( _curl, CURLOPT_DEBUGDATA, &_curlDebug);
564  }
565  }
566 
567  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
568  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
569  if ( ret != 0 ) {
570  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
571  }
572 
573  SET_OPTION(CURLOPT_FAILONERROR, 1L);
574  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
575 
576  // create non persistant settings
577  // so that we don't add headers twice
578  TransferSettings vol_settings(_settings);
579 
580  // add custom headers
581  vol_settings.addHeader(anonymousIdHeader());
582  vol_settings.addHeader(distributionFlavorHeader());
583  vol_settings.addHeader("Pragma:");
584 
585  _settings.setTimeout(ZConfig::instance().download_transfer_timeout());
587 
589 
590  // fill some settings from url query parameters
591  try
592  {
594  }
595  catch ( const MediaException &e )
596  {
597  disconnectFrom();
598  ZYPP_RETHROW(e);
599  }
600  // if the proxy was not set (or explicitly unset) by url, then look...
601  if ( _settings.proxy().empty() )
602  {
603  // ...at the system proxy settings
605  }
606 
610  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
611  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
612  // just in case curl does not trigger its progress callback frequently
613  // enough.
614  if ( _settings.timeout() )
615  {
616  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
617  }
618 
619  // follow any Location: header that the server sends as part of
620  // an HTTP header (#113275)
621  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
622  // 3 redirects seem to be too few in some cases (bnc #465532)
623  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
624 
625  if ( _url.getScheme() == "https" )
626  {
627 #if CURLVERSION_AT_LEAST(7,19,4)
628  // restrict following of redirections from https to https only
629  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
630 #endif
631 
634  {
635  SET_OPTION(CURLOPT_CAPATH, _settings.certificateAuthoritiesPath().c_str());
636  }
637 
638  if( ! _settings.clientCertificatePath().empty() )
639  {
640  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
641  }
642  if( ! _settings.clientKeyPath().empty() )
643  {
644  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
645  }
646 
647 #ifdef CURLSSLOPT_ALLOW_BEAST
648  // see bnc#779177
649  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
650  if ( ret != 0 ) {
651  disconnectFrom();
653  }
654 #endif
655  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
656  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
657  // bnc#903405 - POODLE: libzypp should only talk TLS
658  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
659  }
660 
661  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
662 
663  /*---------------------------------------------------------------*
664  CURLOPT_USERPWD: [user name]:[password]
665 
666  Url::username/password -> CURLOPT_USERPWD
667  If not provided, anonymous FTP identification
668  *---------------------------------------------------------------*/
669 
670  if ( _settings.userPassword().size() )
671  {
672  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
673  string use_auth = _settings.authType();
674  if (use_auth.empty())
675  use_auth = "digest,basic"; // our default
676  long auth = CurlAuthData::auth_type_str2long(use_auth);
677  if( auth != CURLAUTH_NONE)
678  {
679  DBG << "Enabling HTTP authentication methods: " << use_auth
680  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
681  SET_OPTION(CURLOPT_HTTPAUTH, auth);
682  }
683  }
684 
685  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
686  {
687  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
688  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
689  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
690  /*---------------------------------------------------------------*
691  * CURLOPT_PROXYUSERPWD: [user name]:[password]
692  *
693  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
694  * If not provided, $HOME/.curlrc is evaluated
695  *---------------------------------------------------------------*/
696 
697  string proxyuserpwd = _settings.proxyUserPassword();
698 
699  if ( proxyuserpwd.empty() )
700  {
701  CurlConfig curlconf;
702  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
703  if ( curlconf.proxyuserpwd.empty() )
704  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
705  else
706  {
707  proxyuserpwd = curlconf.proxyuserpwd;
708  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
709  }
710  }
711  else
712  {
713  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
714  }
715 
716  if ( ! proxyuserpwd.empty() )
717  {
718  SET_OPTION(CURLOPT_PROXYUSERPWD, unEscape( proxyuserpwd ).c_str());
719  }
720  }
721 #if CURLVERSION_AT_LEAST(7,19,4)
722  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
723  {
724  // Explicitly disabled in URL (see fillSettingsFromUrl()).
725  // This should also prevent libcurl from looking into the environment.
726  DBG << "Proxy: explicitly NOPROXY" << endl;
727  SET_OPTION(CURLOPT_NOPROXY, "*");
728  }
729 #endif
730  else
731  {
732  DBG << "Proxy: not explicitly set" << endl;
733  DBG << "Proxy: libcurl may look into the environment" << endl;
734  }
735 
737  if ( _settings.minDownloadSpeed() != 0 )
738  {
739  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
740  // default to 10 seconds at low speed
741  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
742  }
743 
744 #if CURLVERSION_AT_LEAST(7,15,5)
745  if ( _settings.maxDownloadSpeed() != 0 )
746  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
747 #endif
748 
749  /*---------------------------------------------------------------*
750  *---------------------------------------------------------------*/
751 
752  _currentCookieFile = _cookieFile.asString();
753  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
754  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
755  else
756  MIL << "No cookies requested" << endl;
757  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
758  SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
759  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
760 
761 #if CURLVERSION_AT_LEAST(7,18,0)
762  // bnc #306272
763  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
764 #endif
765  // append settings custom headers to curl
766  for ( TransferSettings::Headers::const_iterator it = vol_settings.headersBegin();
767  it != vol_settings.headersEnd();
768  ++it )
769  {
770  // MIL << "HEADER " << *it << std::endl;
771 
772  _customHeaders = curl_slist_append(_customHeaders, it->c_str());
773  if ( !_customHeaders )
775  }
776 
777  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
778 }
779 
781 
782 
783 void MediaCurl::attachTo (bool next)
784 {
785  if ( next )
787 
788  if ( !_url.isValid() )
790 
793  {
794  std::string mountpoint = createAttachPoint().asString();
795 
796  if( mountpoint.empty())
798 
799  setAttachPoint( mountpoint, true);
800  }
801 
802  disconnectFrom(); // clean _curl if needed
803  _curl = curl_easy_init();
804  if ( !_curl ) {
806  }
807  try
808  {
809  setupEasy();
810  }
811  catch (Exception & ex)
812  {
813  disconnectFrom();
814  ZYPP_RETHROW(ex);
815  }
816 
817  // FIXME: need a derived class to propelly compare url's
819  setMediaSource(media);
820 }
821 
822 bool
823 MediaCurl::checkAttachPoint(const Pathname &apoint) const
824 {
825  return MediaHandler::checkAttachPoint( apoint, true, true);
826 }
827 
829 
831 {
832  if ( _customHeaders )
833  {
834  curl_slist_free_all(_customHeaders);
835  _customHeaders = 0L;
836  }
837 
838  if ( _curl )
839  {
840  curl_easy_cleanup( _curl );
841  _curl = NULL;
842  }
843 }
844 
846 
847 void MediaCurl::releaseFrom( const std::string & ejectDev )
848 {
849  disconnect();
850 }
851 
852 Url MediaCurl::getFileUrl( const Pathname & filename_r ) const
853 {
854  // Simply extend the URLs pathname. An 'absolute' URL path
855  // is achieved by encoding the leading '/' in an URL path:
856  // URL: ftp://user@server -> ~user
857  // URL: ftp://user@server/ -> ~user
858  // URL: ftp://user@server// -> ~user
859  // URL: ftp://user@server/%2F -> /
860  // ^- this '/' is just a separator
861  Url newurl( _url );
862  newurl.setPathName( ( Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) );
863  return newurl;
864 }
865 
867 
868 void MediaCurl::getFile( const Pathname & filename ) const
869 {
870  // Use absolute file name to prevent access of files outside of the
871  // hierarchy below the attach point.
872  getFileCopy(filename, localPath(filename).absolutename());
873 }
874 
876 
877 void MediaCurl::getFileCopy( const Pathname & filename , const Pathname & target) const
878 {
880 
881  Url fileurl(getFileUrl(filename));
882 
883  bool retry = false;
884 
885  do
886  {
887  try
888  {
889  doGetFileCopy(filename, target, report);
890  retry = false;
891  }
892  // retry with proper authentication data
893  catch (MediaUnauthorizedException & ex_r)
894  {
895  if(authenticate(ex_r.hint(), !retry))
896  retry = true;
897  else
898  {
899  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
900  ZYPP_RETHROW(ex_r);
901  }
902  }
903  // unexpected exception
904  catch (MediaException & excpt_r)
905  {
906  // FIXME: error number fix
907  report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory());
908  ZYPP_RETHROW(excpt_r);
909  }
910  }
911  while (retry);
912 
913  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
914 }
915 
917 
918 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
919 {
920  bool retry = false;
921 
922  do
923  {
924  try
925  {
926  return doGetDoesFileExist( filename );
927  }
928  // authentication problem, retry with proper authentication data
929  catch (MediaUnauthorizedException & ex_r)
930  {
931  if(authenticate(ex_r.hint(), !retry))
932  retry = true;
933  else
934  ZYPP_RETHROW(ex_r);
935  }
936  // unexpected exception
937  catch (MediaException & excpt_r)
938  {
939  ZYPP_RETHROW(excpt_r);
940  }
941  }
942  while (retry);
943 
944  return false;
945 }
946 
948 
949 void MediaCurl::evaluateCurlCode( const Pathname &filename,
950  CURLcode code,
951  bool timeout_reached ) const
952 {
953  if ( code != 0 )
954  {
955  Url url;
956  if (filename.empty())
957  url = _url;
958  else
959  url = getFileUrl(filename);
960  std::string err;
961  try
962  {
963  switch ( code )
964  {
965  case CURLE_UNSUPPORTED_PROTOCOL:
966  case CURLE_URL_MALFORMAT:
967  case CURLE_URL_MALFORMAT_USER:
968  err = " Bad URL";
969  break;
970  case CURLE_LOGIN_DENIED:
971  ZYPP_THROW(
972  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
973  break;
974  case CURLE_HTTP_RETURNED_ERROR:
975  {
976  long httpReturnCode = 0;
977  CURLcode infoRet = curl_easy_getinfo( _curl,
978  CURLINFO_RESPONSE_CODE,
979  &httpReturnCode );
980  if ( infoRet == CURLE_OK )
981  {
982  string msg = "HTTP response: " + str::numstring( httpReturnCode );
983  switch ( httpReturnCode )
984  {
985  case 401:
986  {
987  string auth_hint = getAuthHint();
988 
989  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
990  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
991 
993  url, "Login failed.", _curlError, auth_hint
994  ));
995  }
996 
997  case 503: // service temporarily unavailable (bnc #462545)
999  case 504: // gateway timeout
1001  case 403:
1002  {
1003  string msg403;
1004  if (url.asString().find("novell.com") != string::npos)
1005  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
1006  ZYPP_THROW(MediaForbiddenException(url, msg403));
1007  }
1008  case 404:
1010  }
1011 
1012  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
1014  }
1015  else
1016  {
1017  string msg = "Unable to retrieve HTTP response:";
1018  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
1020  }
1021  }
1022  break;
1023  case CURLE_FTP_COULDNT_RETR_FILE:
1024 #if CURLVERSION_AT_LEAST(7,16,0)
1025  case CURLE_REMOTE_FILE_NOT_FOUND:
1026 #endif
1027  case CURLE_FTP_ACCESS_DENIED:
1028  case CURLE_TFTP_NOTFOUND:
1029  err = "File not found";
1031  break;
1032  case CURLE_BAD_PASSWORD_ENTERED:
1033  case CURLE_FTP_USER_PASSWORD_INCORRECT:
1034  err = "Login failed";
1035  break;
1036  case CURLE_COULDNT_RESOLVE_PROXY:
1037  case CURLE_COULDNT_RESOLVE_HOST:
1038  case CURLE_COULDNT_CONNECT:
1039  case CURLE_FTP_CANT_GET_HOST:
1040  err = "Connection failed";
1041  break;
1042  case CURLE_WRITE_ERROR:
1043  err = "Write error";
1044  break;
1045  case CURLE_PARTIAL_FILE:
1046  case CURLE_OPERATION_TIMEDOUT:
1047  timeout_reached = true; // fall though to TimeoutException
1048  // fall though...
1049  case CURLE_ABORTED_BY_CALLBACK:
1050  if( timeout_reached )
1051  {
1052  err = "Timeout reached";
1054  }
1055  else
1056  {
1057  err = "User abort";
1058  }
1059  break;
1060  case CURLE_SSL_PEER_CERTIFICATE:
1061  default:
1062  err = "Unrecognized error";
1063  break;
1064  }
1065 
1066  // uhm, no 0 code but unknown curl exception
1068  }
1069  catch (const MediaException & excpt_r)
1070  {
1071  ZYPP_RETHROW(excpt_r);
1072  }
1073  }
1074  else
1075  {
1076  // actually the code is 0, nothing happened
1077  }
1078 }
1079 
1081 
1082 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
1083 {
1084  DBG << filename.asString() << endl;
1085 
1086  if(!_url.isValid())
1088 
1089  if(_url.getHost().empty())
1091 
1092  Url url(getFileUrl(filename));
1093 
1094  DBG << "URL: " << url.asString() << endl;
1095  // Use URL without options and without username and passwd
1096  // (some proxies dislike them in the URL).
1097  // Curl seems to need the just scheme, hostname and a path;
1098  // the rest was already passed as curl options (in attachTo).
1099  Url curlUrl( clearQueryString(url) );
1100 
1101  //
1102  // See also Bug #154197 and ftp url definition in RFC 1738:
1103  // The url "ftp://user@host/foo/bar/file" contains a path,
1104  // that is relative to the user's home.
1105  // The url "ftp://user@host//foo/bar/file" (or also with
1106  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1107  // contains an absolute path.
1108  //
1109  string urlBuffer( curlUrl.asString());
1110  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1111  urlBuffer.c_str() );
1112  if ( ret != 0 ) {
1114  }
1115 
1116  // instead of returning no data with NOBODY, we return
1117  // little data, that works with broken servers, and
1118  // works for ftp as well, because retrieving only headers
1119  // ftp will return always OK code ?
1120  // See http://curl.haxx.se/docs/knownbugs.html #58
1121  if ( (_url.getScheme() == "http" || _url.getScheme() == "https") &&
1123  ret = curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1124  else
1125  ret = curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1126 
1127  if ( ret != 0 ) {
1128  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1129  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1130  /* yes, this is why we never got to get NOBODY working before,
1131  because setting it changes this option too, and we also
1132  need to reset it
1133  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1134  */
1135  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1137  }
1138 
1139  FILE *file = ::fopen( "/dev/null", "w" );
1140  if ( !file ) {
1141  ERR << "fopen failed for /dev/null" << endl;
1142  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1143  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1144  /* yes, this is why we never got to get NOBODY working before,
1145  because setting it changes this option too, and we also
1146  need to reset it
1147  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1148  */
1149  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1150  if ( ret != 0 ) {
1152  }
1153  ZYPP_THROW(MediaWriteException("/dev/null"));
1154  }
1155 
1156  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1157  if ( ret != 0 ) {
1158  ::fclose(file);
1159  std::string err( _curlError);
1160  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1161  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1162  /* yes, this is why we never got to get NOBODY working before,
1163  because setting it changes this option too, and we also
1164  need to reset it
1165  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1166  */
1167  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1168  if ( ret != 0 ) {
1170  }
1172  }
1173 
1174  CURLcode ok = curl_easy_perform( _curl );
1175  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1176 
1177  // reset curl settings
1178  if ( _url.getScheme() == "http" || _url.getScheme() == "https" )
1179  {
1180  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1181  if ( ret != 0 ) {
1183  }
1184 
1185  /* yes, this is why we never got to get NOBODY working before,
1186  because setting it changes this option too, and we also
1187  need to reset it
1188  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1189  */
1190  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L);
1191  if ( ret != 0 ) {
1193  }
1194 
1195  }
1196  else
1197  {
1198  // for FTP we set different options
1199  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL);
1200  if ( ret != 0 ) {
1202  }
1203  }
1204 
1205  // if the code is not zero, close the file
1206  if ( ok != 0 )
1207  ::fclose(file);
1208 
1209  // as we are not having user interaction, the user can't cancel
1210  // the file existence checking, a callback or timeout return code
1211  // will be always a timeout.
1212  try {
1213  evaluateCurlCode( filename, ok, true /* timeout */);
1214  }
1215  catch ( const MediaFileNotFoundException &e ) {
1216  // if the file did not exist then we can return false
1217  return false;
1218  }
1219  catch ( const MediaException &e ) {
1220  // some error, we are not sure about file existence, rethrw
1221  ZYPP_RETHROW(e);
1222  }
1223  // exists
1224  return ( ok == CURLE_OK );
1225 }
1226 
1228 
1229 
1230 #if DETECT_DIR_INDEX
1231 bool MediaCurl::detectDirIndex() const
1232 {
1233  if(_url.getScheme() != "http" && _url.getScheme() != "https")
1234  return false;
1235  //
1236  // try to check the effective url and set the not_a_file flag
1237  // if the url path ends with a "/", what usually means, that
1238  // we've received a directory index (index.html content).
1239  //
1240  // Note: This may be dangerous and break file retrieving in
1241  // case of some server redirections ... ?
1242  //
1243  bool not_a_file = false;
1244  char *ptr = NULL;
1245  CURLcode ret = curl_easy_getinfo( _curl,
1246  CURLINFO_EFFECTIVE_URL,
1247  &ptr);
1248  if ( ret == CURLE_OK && ptr != NULL)
1249  {
1250  try
1251  {
1252  Url eurl( ptr);
1253  std::string path( eurl.getPathName());
1254  if( !path.empty() && path != "/" && *path.rbegin() == '/')
1255  {
1256  DBG << "Effective url ("
1257  << eurl
1258  << ") seems to provide the index of a directory"
1259  << endl;
1260  not_a_file = true;
1261  }
1262  }
1263  catch( ... )
1264  {}
1265  }
1266  return not_a_file;
1267 }
1268 #endif
1269 
1271 
1272 void MediaCurl::doGetFileCopy( const Pathname & filename , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1273 {
1274  Pathname dest = target.absolutename();
1275  if( assert_dir( dest.dirname() ) )
1276  {
1277  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1278  Url url(getFileUrl(filename));
1279  ZYPP_THROW( MediaSystemException(url, "System error on " + dest.dirname().asString()) );
1280  }
1281  string destNew = target.asString() + ".new.zypp.XXXXXX";
1282  char *buf = ::strdup( destNew.c_str());
1283  if( !buf)
1284  {
1285  ERR << "out of memory for temp file name" << endl;
1286  Url url(getFileUrl(filename));
1287  ZYPP_THROW(MediaSystemException(url, "out of memory for temp file name"));
1288  }
1289 
1290  int tmp_fd = ::mkostemp( buf, O_CLOEXEC );
1291  if( tmp_fd == -1)
1292  {
1293  free( buf);
1294  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1295  ZYPP_THROW(MediaWriteException(destNew));
1296  }
1297  destNew = buf;
1298  free( buf);
1299 
1300  FILE *file = ::fdopen( tmp_fd, "we" );
1301  if ( !file ) {
1302  ::close( tmp_fd);
1303  filesystem::unlink( destNew );
1304  ERR << "fopen failed for file '" << destNew << "'" << endl;
1305  ZYPP_THROW(MediaWriteException(destNew));
1306  }
1307 
1308  DBG << "dest: " << dest << endl;
1309  DBG << "temp: " << destNew << endl;
1310 
1311  // set IFMODSINCE time condition (no download if not modified)
1312  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1313  {
1314  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1315  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1316  }
1317  else
1318  {
1319  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1320  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1321  }
1322  try
1323  {
1324  doGetFileCopyFile(filename, dest, file, report, options);
1325  }
1326  catch (Exception &e)
1327  {
1328  ::fclose( file );
1329  filesystem::unlink( destNew );
1330  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1331  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1332  ZYPP_RETHROW(e);
1333  }
1334 
1335  long httpReturnCode = 0;
1336  CURLcode infoRet = curl_easy_getinfo(_curl,
1337  CURLINFO_RESPONSE_CODE,
1338  &httpReturnCode);
1339  bool modified = true;
1340  if (infoRet == CURLE_OK)
1341  {
1342  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1343  if ( httpReturnCode == 304
1344  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1345  {
1346  DBG << " Not modified.";
1347  modified = false;
1348  }
1349  DBG << endl;
1350  }
1351  else
1352  {
1353  WAR << "Could not get the reponse code." << endl;
1354  }
1355 
1356  if (modified || infoRet != CURLE_OK)
1357  {
1358  // apply umask
1359  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1360  {
1361  ERR << "Failed to chmod file " << destNew << endl;
1362  }
1363  if (::fclose( file ))
1364  {
1365  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1366  ZYPP_THROW(MediaWriteException(destNew));
1367  }
1368  // move the temp file into dest
1369  if ( rename( destNew, dest ) != 0 ) {
1370  ERR << "Rename failed" << endl;
1372  }
1373  }
1374  else
1375  {
1376  // close and remove the temp file
1377  ::fclose( file );
1378  filesystem::unlink( destNew );
1379  }
1380 
1381  DBG << "done: " << PathInfo(dest) << endl;
1382 }
1383 
1385 
1386 void MediaCurl::doGetFileCopyFile( const Pathname & filename , const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1387 {
1388  DBG << filename.asString() << endl;
1389 
1390  if(!_url.isValid())
1392 
1393  if(_url.getHost().empty())
1395 
1396  Url url(getFileUrl(filename));
1397 
1398  DBG << "URL: " << url.asString() << endl;
1399  // Use URL without options and without username and passwd
1400  // (some proxies dislike them in the URL).
1401  // Curl seems to need the just scheme, hostname and a path;
1402  // the rest was already passed as curl options (in attachTo).
1403  Url curlUrl( clearQueryString(url) );
1404 
1405  //
1406  // See also Bug #154197 and ftp url definition in RFC 1738:
1407  // The url "ftp://user@host/foo/bar/file" contains a path,
1408  // that is relative to the user's home.
1409  // The url "ftp://user@host//foo/bar/file" (or also with
1410  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1411  // contains an absolute path.
1412  //
1413  string urlBuffer( curlUrl.asString());
1414  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1415  urlBuffer.c_str() );
1416  if ( ret != 0 ) {
1418  }
1419 
1420  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1421  if ( ret != 0 ) {
1423  }
1424 
1425  // Set callback and perform.
1426  ProgressData progressData(_curl, _settings.timeout(), url, &report);
1427  if (!(options & OPTION_NO_REPORT_START))
1428  report->start(url, dest);
1429  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1430  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1431  }
1432 
1433  ret = curl_easy_perform( _curl );
1434 #if CURLVERSION_AT_LEAST(7,19,4)
1435  // bnc#692260: If the client sends a request with an If-Modified-Since header
1436  // with a future date for the server, the server may respond 200 sending a
1437  // zero size file.
1438  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1439  if ( ftell(file) == 0 && ret == 0 )
1440  {
1441  long httpReturnCode = 33;
1442  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1443  {
1444  long conditionUnmet = 33;
1445  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1446  {
1447  WAR << "TIMECONDITION unmet - retry without." << endl;
1448  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1449  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1450  ret = curl_easy_perform( _curl );
1451  }
1452  }
1453  }
1454 #endif
1455 
1456  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1457  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1458  }
1459 
1460  if ( ret != 0 )
1461  {
1462  ERR << "curl error: " << ret << ": " << _curlError
1463  << ", temp file size " << ftell(file)
1464  << " bytes." << endl;
1465 
1466  // the timeout is determined by the progress data object
1467  // which holds whether the timeout was reached or not,
1468  // otherwise it would be a user cancel
1469  try {
1470  evaluateCurlCode( filename, ret, progressData.reached);
1471  }
1472  catch ( const MediaException &e ) {
1473  // some error, we are not sure about file existence, rethrw
1474  ZYPP_RETHROW(e);
1475  }
1476  }
1477 
1478 #if DETECT_DIR_INDEX
1479  if (!ret && detectDirIndex())
1480  {
1482  }
1483 #endif // DETECT_DIR_INDEX
1484 }
1485 
1487 
1488 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1489 {
1490  filesystem::DirContent content;
1491  getDirInfo( content, dirname, /*dots*/false );
1492 
1493  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1494  Pathname filename = dirname + it->name;
1495  int res = 0;
1496 
1497  switch ( it->type ) {
1498  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1499  case filesystem::FT_FILE:
1500  getFile( filename );
1501  break;
1502  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1503  if ( recurse_r ) {
1504  getDir( filename, recurse_r );
1505  } else {
1506  res = assert_dir( localPath( filename ) );
1507  if ( res ) {
1508  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1509  }
1510  }
1511  break;
1512  default:
1513  // don't provide devices, sockets, etc.
1514  break;
1515  }
1516  }
1517 }
1518 
1520 
1521 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1522  const Pathname & dirname, bool dots ) const
1523 {
1524  getDirectoryYast( retlist, dirname, dots );
1525 }
1526 
1528 
1530  const Pathname & dirname, bool dots ) const
1531 {
1532  getDirectoryYast( retlist, dirname, dots );
1533 }
1534 
1536 
1537 int MediaCurl::progressCallback( void *clientp,
1538  double dltotal, double dlnow,
1539  double ultotal, double ulnow)
1540 {
1541  ProgressData *pdata = reinterpret_cast<ProgressData *>(clientp);
1542  if( pdata)
1543  {
1544  // work around curl bug that gives us old data
1545  long httpReturnCode = 0;
1546  if (curl_easy_getinfo(pdata->curl, CURLINFO_RESPONSE_CODE, &httpReturnCode) != CURLE_OK || httpReturnCode == 0)
1547  return 0;
1548 
1549  time_t now = time(NULL);
1550  if( now > 0)
1551  {
1552  // reset time of last change in case initial time()
1553  // failed or the time was adjusted (goes backward)
1554  if( pdata->ltime <= 0 || pdata->ltime > now)
1555  {
1556  pdata->ltime = now;
1557  }
1558 
1559  // start time counting as soon as first data arrives
1560  // (skip the connection / redirection time at begin)
1561  time_t dif = 0;
1562  if (dlnow > 0 || ulnow > 0)
1563  {
1564  dif = (now - pdata->ltime);
1565  dif = dif > 0 ? dif : 0;
1566 
1567  pdata->secs += dif;
1568  }
1569 
1570  // update the drate_avg and drate_period only after a second has passed
1571  // (this callback is called much more often than a second)
1572  // otherwise the values would be far from accurate when measuring
1573  // the time in seconds
1575 
1576  if ( pdata->secs > 1 && (dif > 0 || dlnow == dltotal ))
1577  pdata->drate_avg = (dlnow / pdata->secs);
1578 
1579  if ( dif > 0 )
1580  {
1581  pdata->drate_period = ((dlnow - pdata->dload_period) / dif);
1582  pdata->dload_period = dlnow;
1583  }
1584  }
1585 
1586  // send progress report first, abort transfer if requested
1587  if( pdata->report)
1588  {
1589  if (!(*(pdata->report))->progress(int( dltotal ? dlnow * 100 / dltotal : 0 ),
1590  pdata->url,
1591  pdata->drate_avg,
1592  pdata->drate_period))
1593  {
1594  return 1; // abort transfer
1595  }
1596  }
1597 
1598  // check if we there is a timeout set
1599  if( pdata->timeout > 0)
1600  {
1601  if( now > 0)
1602  {
1603  bool progress = false;
1604 
1605  // update download data if changed, mark progress
1606  if( dlnow != pdata->dload)
1607  {
1608  progress = true;
1609  pdata->dload = dlnow;
1610  pdata->ltime = now;
1611  }
1612  // update upload data if changed, mark progress
1613  if( ulnow != pdata->uload)
1614  {
1615  progress = true;
1616  pdata->uload = ulnow;
1617  pdata->ltime = now;
1618  }
1619 
1620  if( !progress && (now >= (pdata->ltime + pdata->timeout)))
1621  {
1622  pdata->reached = true;
1623  return 1; // aborts transfer
1624  }
1625  }
1626  }
1627  }
1628  return 0;
1629 }
1630 
1632 {
1633  ProgressData *pdata = reinterpret_cast<ProgressData *>(clientp);
1634  return pdata ? pdata->curl : 0;
1635 }
1636 
1638 
1640 {
1641  long auth_info = CURLAUTH_NONE;
1642 
1643  CURLcode infoRet =
1644  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1645 
1646  if(infoRet == CURLE_OK)
1647  {
1648  return CurlAuthData::auth_type_long2str(auth_info);
1649  }
1650 
1651  return "";
1652 }
1653 
1655 
1656 bool MediaCurl::authenticate(const string & availAuthTypes, bool firstTry) const
1657 {
1659  Target_Ptr target = zypp::getZYpp()->getTarget();
1660  CredentialManager cm(CredManagerOptions(target ? target->root() : ""));
1661  CurlAuthData_Ptr credentials;
1662 
1663  // get stored credentials
1664  AuthData_Ptr cmcred = cm.getCred(_url);
1665 
1666  if (cmcred && firstTry)
1667  {
1668  credentials.reset(new CurlAuthData(*cmcred));
1669  DBG << "got stored credentials:" << endl << *credentials << endl;
1670  }
1671  // if not found, ask user
1672  else
1673  {
1674 
1675  CurlAuthData_Ptr curlcred;
1676  curlcred.reset(new CurlAuthData());
1678 
1679  // preset the username if present in current url
1680  if (!_url.getUsername().empty() && firstTry)
1681  curlcred->setUsername(_url.getUsername());
1682  // if CM has found some credentials, preset the username from there
1683  else if (cmcred)
1684  curlcred->setUsername(cmcred->username());
1685 
1686  // indicate we have no good credentials from CM
1687  cmcred.reset();
1688 
1689  string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1690 
1691  // set available authentication types from the exception
1692  // might be needed in prompt
1693  curlcred->setAuthType(availAuthTypes);
1694 
1695  // ask user
1696  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1697  {
1698  DBG << "callback answer: retry" << endl
1699  << "CurlAuthData: " << *curlcred << endl;
1700 
1701  if (curlcred->valid())
1702  {
1703  credentials = curlcred;
1704  // if (credentials->username() != _url.getUsername())
1705  // _url.setUsername(credentials->username());
1713  }
1714  }
1715  else
1716  {
1717  DBG << "callback answer: cancel" << endl;
1718  }
1719  }
1720 
1721  // set username and password
1722  if (credentials)
1723  {
1724  // HACK, why is this const?
1725  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1726  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1727 
1728  // set username and password
1729  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1731 
1732  // set available authentication types from the exception
1733  if (credentials->authType() == CURLAUTH_NONE)
1734  credentials->setAuthType(availAuthTypes);
1735 
1736  // set auth type (seems this must be set _after_ setting the userpwd)
1737  if (credentials->authType() != CURLAUTH_NONE)
1738  {
1739  // FIXME: only overwrite if not empty?
1740  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1741  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1743  }
1744 
1745  if (!cmcred)
1746  {
1747  credentials->setUrl(_url);
1748  cm.addCred(*credentials);
1749  cm.save();
1750  }
1751 
1752  return true;
1753  }
1754 
1755  return false;
1756 }
1757 
1758 
1759  } // namespace media
1760 } // namespace zypp
1761 //
void setPassword(const std::string &pass, EEncoding eflag=zypp::url::E_DECODED)
Set the password in the URL authority.
Definition: Url.cc:733
std::string userPassword() const
returns the user and password as a user:pass string
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition: PathInfo.cc:320
Interface to gettext.
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:529
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:451
#define MIL
Definition: Logger.h:64
#define CONNECT_TIMEOUT
Definition: MediaCurl.cc:42
bool verifyHostEnabled() const
Whether to verify host for ssl.
Pathname clientKeyPath() const
SSL client key file.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:321
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1656
static ZConfig & instance()
Singleton ctor.
Definition: Resolver.cc:121
virtual void releaseFrom(const std::string &ejectDev)
Call concrete handler to release the media.
Definition: MediaCurl.cc:847
const std::string & msg() const
Return the message string provided to the ctor.
Definition: Exception.h:185
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:32
Flag to request encoded string(s).
Definition: UrlUtils.h:53
long connectTimeout() const
connection timeout
Headers::const_iterator headersEnd() const
end iterators to additional headers
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
Definition: Url.cc:598
void setClientKeyPath(const zypp::Pathname &path)
Sets the SSL client key file.
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:44
TransferSettings & settings()
Definition: MediaCurl.cc:516
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:582
Holds transfer setting.
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:491
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
std::string escape(const C_Str &str_r, const char sep_r)
Escape desired character c using a backslash.
Definition: String.cc:369
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Definition: MediaCurl.cc:1537
void setProxyUsername(const std::string &proxyuser)
sets the proxy user
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
Pathname certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
void setPathParams(const std::string &params)
Set the path parameters.
Definition: Url.cc:780
void setHeadRequestsAllowed(bool allowed)
set whether HEAD requests are allowed
pthread_once_t OnceFlag
The OnceFlag variable type.
Definition: Once.h:32
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:566
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
void setConnectTimeout(long t)
set the connect timeout
void setUsername(const std::string &user, EEncoding eflag=zypp::url::E_DECODED)
Set the username in the URL authority.
Definition: Url.cc:724
double dload
Definition: MediaCurl.cc:189
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:554
#define EXPLICITLY_NO_PROXY
Definition: MediaCurl.cc:45
Convenient building of std::string with boost::format.
Definition: String.h:247
Structure holding values of curlrc options.
Definition: CurlConfig.h:16
bool isValid() const
Verifies the Url.
Definition: Url.cc:483
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
Edition * _value
Definition: SysContent.cc:311
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
std::string _currentCookieFile
Definition: MediaCurl.h:167
void setProxy(const std::string &proxyhost)
proxy to use if it is enabled
void setFragment(const std::string &fragment, EEncoding eflag=zypp::url::E_DECODED)
Set the fragment string in the URL.
Definition: Url.cc:716
#define ERR
Definition: Logger.h:66
void setPassword(const std::string &password)
sets the auth password
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:491
void setUsername(const std::string &username)
sets the auth username
bool headRequestsAllowed() const
whether HEAD requests are allowed
void setAnonymousAuth()
sets anonymous authentication (ie: for ftp)
virtual void getFile(const Pathname &filename) const
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:868
std::string proxy(const Url &url) const
Definition: ProxyInfo.cc:44
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:522
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1639
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:329
void setPathName(const std::string &path, EEncoding eflag=zypp::url::E_DECODED)
Set the path name.
Definition: Url.cc:758
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: CurlConfig.cc:24
void doGetFileCopyFile(const Pathname &srcFilename, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1386
std::string userAgentString() const
user agent string
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \t")
Split line_r into words.
Definition: String.h:518
void setProxyPassword(const std::string &proxypass)
sets the proxy password
Abstract base class for 'physical' MediaHandler like MediaCD, etc.
Definition: MediaHandler.h:45
void callOnce(OnceFlag &flag, void(*func)())
Call once function.
Definition: Once.h:50
void setAuthType(const std::string &authtype)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:221
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:653
const Url _url
Url to handle.
Definition: MediaHandler.h:110
virtual bool getDoesFileExist(const Pathname &filename) const
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:918
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
Definition: PathInfo.cc:667
Just inherits Exception to separate media exceptions.
long timeout
Definition: MediaCurl.cc:175
void disconnect()
Use concrete handler to isconnect media.
do not send a start ProgressReport
Definition: MediaCurl.h:46
#define WAR
Definition: Logger.h:65
TransferSettings _settings
Definition: MediaCurl.h:174
time_t ltime
Definition: MediaCurl.cc:187
bool reached
Definition: MediaCurl.cc:176
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
zypp::Url url
Definition: MediaCurl.cc:192
void setTimeout(long t)
set the transfer timeout
bool useProxyFor(const Url &url_r) const
Return true if enabled and url_r does not match noProxy.
Definition: ProxyInfo.cc:56
#define _(MSG)
Definition: Gettext.h:29
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
static const char *const agentString()
initialized only once, this gets the agent string which also includes the curl version ...
Definition: MediaCurl.cc:424
Pathname localPath(const Pathname &pathname) const
Files provided will be available at 'localPath(filename)'.
std::string proxyuserpwd
Definition: CurlConfig.h:39
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:654
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:823
shared_ptr< CurlAuthData > CurlAuthData_Ptr
virtual void getDir(const Pathname &dirname, bool recurse_r) const
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1488
std::string numstring(char n, int w=0)
Definition: String.h:304
virtual void disconnectFrom()
Definition: MediaCurl.cc:830
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
SolvableIdType size_type
Definition: PoolMember.h:126
bool detectDirIndex() const
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:36
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
void fillSettingsFromUrl(const Url &url, TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: MediaCurl.cc:223
curl_slist * _customHeaders
Definition: MediaCurl.h:173
Headers::const_iterator headersBegin() const
begin iterators to additional headers
void setClientCertificatePath(const zypp::Pathname &path)
Sets the SSL client certificate file.
shared_ptr< AuthData > AuthData_Ptr
Definition: MediaUserAuth.h:69
int rmdir(const Pathname &path)
Like 'rmdir'.
Definition: PathInfo.cc:367
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:444
Pathname attachPoint() const
Return the currently used attach point.
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
Definition: MediaCurl.cc:852
Base class for Exception.
Definition: Exception.h:143
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1521
const std::string & hint() const
comma separated list of available authentication types
static const char *const distributionFlavorHeader()
initialized only once, this gets the distribution flavor from the target, which we pass in the http h...
Definition: MediaCurl.cc:406
void fillSettingsSystemProxy(const Url &url, TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: MediaCurl.cc:360
callback::SendReport< DownloadProgressReport > * report
Definition: MediaCurl.cc:177
void addHeader(const std::string &header)
add a header, on the form "Foo: Bar"
CURL * curl
Definition: MediaCurl.cc:174
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1631
void setCertificateAuthoritiesPath(const zypp::Pathname &path)
Sets the SSL certificate authorities path.
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:444
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:75
virtual void attachTo(bool next=false)
Call concrete handler to attach the media.
Definition: MediaCurl.cc:783
virtual void getFileCopy(const Pathname &srcFilename, const Pathname &targetFilename) const
Definition: MediaCurl.cc:877
double dload_period
Definition: MediaCurl.cc:181
virtual void doGetFileCopy(const Pathname &srcFilename, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1272
static Pathname _cookieFile
Definition: MediaCurl.h:168
double drate_avg
Definition: MediaCurl.cc:185
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:806
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:1082
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:527
std::string authType() const
get the allowed authentication types
double uload
Definition: MediaCurl.cc:191
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
#define TRANSFER_TIMEOUT_MAX
Definition: MediaCurl.cc:43
Curl HTTP authentication data.
Definition: MediaUserAuth.h:74
double drate_period
Definition: MediaCurl.cc:179
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:172
void setVerifyPeerEnabled(bool enabled)
Sets whether to verify host for ssl.
Pathname clientCertificatePath() const
SSL client certificate file.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:949
Url url() const
Url used.
Definition: MediaHandler.h:506
std::string proxy() const
proxy host
bool proxyEnabled() const
proxy is enabled
long secs
Definition: MediaCurl.cc:183
Convenience interface for handling authentication data of media user.
void setVerifyHostEnabled(bool enabled)
Sets whether to verify host for ssl.
Url manipulation class.
Definition: Url.h:87
void setUserAgentString(const std::string &agent)
sets the user agent ie: "Mozilla v3"
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
static const char *const anonymousIdHeader()
initialized only once, this gets the anonymous id from the target, which we pass in the http header ...
Definition: MediaCurl.cc:388
void setProxyEnabled(bool enabled)
whether the proxy is used or not
std::string username() const
auth username
#define DBG
Definition: Logger.h:63
std::string getPassword(EEncoding eflag=zypp::url::E_DECODED) const
Returns the password from the URL authority.
Definition: Url.cc:574
void delQueryParam(const std::string &param)
remove the specified query parameter.
Definition: Url.cc:834
std::string proxyUsername() const
proxy auth username
long timeout() const
transfer timeout