Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

# 

#    Licensed under the Apache License, Version 2.0 (the "License"); you may 

#    not use this file except in compliance with the License. You may obtain 

#    a copy of the License at 

# 

#         http://www.apache.org/licenses/LICENSE-2.0 

# 

#    Unless required by applicable law or agreed to in writing, software 

#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 

#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 

#    License for the specific language governing permissions and limitations 

#    under the License. 

 

"""Utility for fetching a resource (e.g. a manifest) from a URL.""" 

 

from oslo_config import cfg 

import requests 

from requests import exceptions 

from six.moves import urllib 

 

from magnum.common import exception 

from magnum.openstack.common._i18n import _ 

from magnum.openstack.common._i18n import _LI 

from magnum.openstack.common import log as logging 

 

URLFETCH_OPTS = [ 

    cfg.IntOpt('max_manifest_size', 

               default=524288, 

               help=_('Maximum raw byte size of any manifest.')) 

] 

 

cfg.CONF.register_opts(URLFETCH_OPTS) 

 

LOG = logging.getLogger(__name__) 

 

 

class URLFetchError(exception.Invalid, IOError): 

    pass 

 

 

def get(url, allowed_schemes=('http', 'https')): 

    """Get the data at the specified URL. 

 

    The URL must use the http: or https: schemes. 

    The file: scheme is also supported if you override 

    the allowed_schemes argument. 

    Raise an IOError if getting the data fails. 

    """ 

    LOG.info(_LI('Fetching data from %s'), url) 

 

    components = urllib.parse.urlparse(url) 

 

    if components.scheme not in allowed_schemes: 

        raise URLFetchError(_('Invalid URL scheme %s') % components.scheme) 

 

57    if components.scheme == 'file': 

        try: 

            return urllib.request.urlopen(url).read() 

        except urllib.error.URLError as uex: 

            raise URLFetchError(_('Failed to retrieve manifest: %s') % uex) 

 

    try: 

        resp = requests.get(url, stream=True) 

        resp.raise_for_status() 

 

        # We cannot use resp.text here because it would download the 

        # entire file, and a large enough file would bring down the 

        # engine.  The 'Content-Length' header could be faked, so it's 

        # necessary to download the content in chunks to until 

        # max_manifest_size is reached.  The chunk_size we use needs 

        # to balance CPU-intensive string concatenation with accuracy 

        # (eg. it's possible to fetch 1000 bytes greater than 

        # max_manifest_size with a chunk_size of 1000). 

        reader = resp.iter_content(chunk_size=1000) 

        result = "" 

        for chunk in reader: 

            result += chunk 

            if len(result) > cfg.CONF.max_manifest_size: 

                raise URLFetchError("Manifest exceeds maximum allowed size (%s" 

                                    " bytes)" % cfg.CONF.max_manifest_size) 

        return result 

 

    except exceptions.RequestException as ex: 

        raise URLFetchError(_('Failed to retrieve manifest: %s') % ex)