Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

# Copyright 2010 United States Government as represented by the 

# Administrator of the National Aeronautics and Space Administration. 

# Copyright 2011 Justin Santa Barbara 

# Copyright (c) 2012 NTT DOCOMO, INC. 

# All Rights Reserved. 

# 

#    Licensed under the Apache License, Version 2.0 (the "License"); you may 

#    not use this file except in compliance with the License. You may obtain 

#    a copy of the License at 

# 

#         http://www.apache.org/licenses/LICENSE-2.0 

# 

#    Unless required by applicable law or agreed to in writing, software 

#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 

#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 

#    License for the specific language governing permissions and limitations 

#    under the License. 

 

"""Utilities and helper functions.""" 

 

import contextlib 

import errno 

import hashlib 

import os 

import pecan 

import random 

import re 

import shutil 

import tempfile 

import uuid 

 

import netaddr 

from oslo_concurrency import processutils 

from oslo_config import cfg 

from oslo_utils import excutils 

import paramiko 

import six 

 

from magnum.common import exception 

from magnum.openstack.common._i18n import _ 

from magnum.openstack.common._i18n import _LE 

from magnum.openstack.common._i18n import _LW 

from magnum.openstack.common import log as logging 

 

UTILS_OPTS = [ 

    cfg.StrOpt('rootwrap_config', 

               default="/etc/magnum/rootwrap.conf", 

               help='Path to the rootwrap configuration file to use for ' 

                    'running commands as root.'), 

    cfg.StrOpt('tempdir', 

               help='Explicitly specify the temporary working directory.'), 

] 

 

CONF = cfg.CONF 

CONF.register_opts(UTILS_OPTS) 

 

LOG = logging.getLogger(__name__) 

 

 

def _get_root_helper(): 

    return 'sudo magnum-rootwrap %s' % CONF.rootwrap_config 

 

 

def execute(*cmd, **kwargs): 

    """Convenience wrapper around oslo's execute() method. 

 

    :param cmd: Passed to processutils.execute. 

    :param use_standard_locale: True | False. Defaults to False. If set to 

                                True, execute command with standard locale 

                                added to environment variables. 

    :returns: (stdout, stderr) from process execution 

    :raises: UnknownArgumentError 

    :raises: ProcessExecutionError 

    """ 

 

    use_standard_locale = kwargs.pop('use_standard_locale', False) 

    if use_standard_locale: 

        env = kwargs.pop('env_variables', os.environ.copy()) 

        env['LC_ALL'] = 'C' 

        kwargs['env_variables'] = env 

    if kwargs.get('run_as_root') and 'root_helper' not in kwargs: 

        kwargs['root_helper'] = _get_root_helper() 

    result = processutils.execute(*cmd, **kwargs) 

    LOG.debug('Execution completed, command line is "%s"', 

              ' '.join(map(str, cmd))) 

    LOG.debug('Command stdout is: "%s"' % result[0]) 

    LOG.debug('Command stderr is: "%s"' % result[1]) 

    return result 

 

 

def trycmd(*args, **kwargs): 

    """Convenience wrapper around oslo's trycmd() method.""" 

    if kwargs.get('run_as_root') and 'root_helper' not in kwargs: 

        kwargs['root_helper'] = _get_root_helper() 

    return processutils.trycmd(*args, **kwargs) 

 

 

def ssh_connect(connection): 

    """Method to connect to a remote system using ssh protocol. 

 

    :param connection: a dict of connection parameters. 

    :returns: paramiko.SSHClient -- an active ssh connection. 

    :raises: SSHConnectFailed 

 

    """ 

    try: 

        ssh = paramiko.SSHClient() 

        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 

        key_contents = connection.get('key_contents') 

        if key_contents: 

            data = six.moves.StringIO(key_contents) 

            if "BEGIN RSA PRIVATE" in key_contents: 

                pkey = paramiko.RSAKey.from_private_key(data) 

            elif "BEGIN DSA PRIVATE" in key_contents: 

                pkey = paramiko.DSSKey.from_private_key(data) 

            else: 

                # Can't include the key contents - secure material. 

                raise ValueError(_("Invalid private key")) 

        else: 

            pkey = None 

        ssh.connect(connection.get('host'), 

                    username=connection.get('username'), 

                    password=connection.get('password'), 

                    port=connection.get('port', 22), 

                    pkey=pkey, 

                    key_filename=connection.get('key_filename'), 

                    timeout=connection.get('timeout', 10)) 

 

        # send TCP keepalive packets every 20 seconds 

        ssh.get_transport().set_keepalive(20) 

    except Exception as e: 

        LOG.debug("SSH connect failed: %s" % e) 

        raise exception.SSHConnectFailed(host=connection.get('host')) 

 

    return ssh 

 

 

def generate_uid(topic, size=8): 

    characters = '01234567890abcdefghijklmnopqrstuvwxyz' 

    choices = [random.choice(characters) for _x in range(size)] 

    return '%s-%s' % (topic, ''.join(choices)) 

 

 

def random_alnum(size=32): 

    characters = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' 

    return ''.join(random.choice(characters) for _ in range(size)) 

 

 

def delete_if_exists(pathname): 

    """delete a file, but ignore file not found error.""" 

 

    try: 

        os.unlink(pathname) 

    except OSError as e: 

        if e.errno == errno.ENOENT: 

            return 

        else: 

            raise 

 

 

def is_int_like(val): 

    """Check if a value looks like an int.""" 

    try: 

        return str(int(val)) == str(val) 

    except Exception: 

        return False 

 

 

def is_valid_boolstr(val): 

    """Check if the provided string is a valid bool string or not.""" 

    boolstrs = ('true', 'false', 'yes', 'no', 'y', 'n', '1', '0') 

    return str(val).lower() in boolstrs 

 

 

def is_valid_mac(address): 

    """Verify the format of a MAC address. 

 

    Check if a MAC address is valid and contains six octets. Accepts 

    colon-separated format only. 

 

    :param address: MAC address to be validated. 

    :returns: True if valid. False if not. 

 

    """ 

    m = "[0-9a-f]{2}(:[0-9a-f]{2}){5}$" 

    if isinstance(address, six.string_types) and re.match(m, address.lower()): 

        return True 

    return False 

 

 

def validate_and_normalize_mac(address): 

    """Validate a MAC address and return normalized form. 

 

    Checks whether the supplied MAC address is formally correct and 

    normalize it to all lower case. 

 

    :param address: MAC address to be validated and normalized. 

    :returns: Normalized and validated MAC address. 

    :raises: InvalidMAC If the MAC address is not valid. 

 

    """ 

    if not is_valid_mac(address): 

        raise exception.InvalidMAC(mac=address) 

    return address.lower() 

 

 

def is_valid_ipv4(address): 

    """Verify that address represents a valid IPv4 address.""" 

    try: 

        return netaddr.valid_ipv4(address) 

    except Exception: 

        return False 

 

 

def is_valid_ipv6(address): 

    try: 

        return netaddr.valid_ipv6(address) 

    except Exception: 

        return False 

 

 

def is_valid_ipv6_cidr(address): 

    try: 

        str(netaddr.IPNetwork(address, version=6).cidr) 

        return True 

    except Exception: 

        return False 

 

 

def get_shortened_ipv6(address): 

    addr = netaddr.IPAddress(address, version=6) 

    return str(addr.ipv6()) 

 

 

def get_shortened_ipv6_cidr(address): 

    net = netaddr.IPNetwork(address, version=6) 

    return str(net.cidr) 

 

 

def is_valid_cidr(address): 

    """Check if the provided ipv4 or ipv6 address is a valid CIDR address.""" 

    try: 

        # Validate the correct CIDR Address 

        netaddr.IPNetwork(address) 

    except netaddr.core.AddrFormatError: 

        return False 

    except UnboundLocalError: 

        # NOTE(MotoKen): work around bug in netaddr 0.7.5 (see detail in 

        # https://github.com/drkjam/netaddr/issues/2) 

        return False 

 

    # Prior validation partially verify /xx part 

    # Verify it here 

    ip_segment = address.split('/') 

 

    if (len(ip_segment) <= 1 or 

        ip_segment[1] == ''): 

        return False 

 

    return True 

 

 

def get_ip_version(network): 

    """Returns the IP version of a network (IPv4 or IPv6). 

 

    :raises: AddrFormatError if invalid network. 

    """ 

    if netaddr.IPNetwork(network).version == 6: 

        return "IPv6" 

    elif netaddr.IPNetwork(network).version == 4: 

        return "IPv4" 

 

 

def convert_to_list_dict(lst, label): 

    """Convert a value or list into a list of dicts.""" 

    if not lst: 

        return None 

    if not isinstance(lst, list): 

        lst = [lst] 

    return [{label: x} for x in lst] 

 

 

def sanitize_hostname(hostname): 

    """Return a hostname which conforms to RFC-952 and RFC-1123 specs.""" 

    if isinstance(hostname, six.text_type): 

        hostname = hostname.encode('latin-1', 'ignore') 

 

    hostname = re.sub('[ _]', '-', hostname) 

    hostname = re.sub('[^\w.-]+', '', hostname) 

    hostname = hostname.lower() 

    hostname = hostname.strip('.-') 

 

    return hostname 

 

 

def read_cached_file(filename, cache_info, reload_func=None): 

    """Read from a file if it has been modified. 

 

    :param cache_info: dictionary to hold opaque cache. 

    :param reload_func: optional function to be called with data when 

                        file is reloaded due to a modification. 

 

    :returns: data from file 

 

    """ 

    mtime = os.path.getmtime(filename) 

    if not cache_info or mtime != cache_info.get('mtime'): 

        LOG.debug("Reloading cached file %s" % filename) 

        with open(filename) as fap: 

            cache_info['data'] = fap.read() 

        cache_info['mtime'] = mtime 

314        if reload_func: 

            reload_func(cache_info['data']) 

    return cache_info['data'] 

 

 

def file_open(*args, **kwargs): 

    """Open file 

 

    see built-in file() documentation for more details 

 

    Note: The reason this is kept in a separate module is to easily 

          be able to provide a stub module that doesn't alter system 

          state at all (for unit tests) 

    """ 

    return file(*args, **kwargs) 

 

 

def hash_file(file_like_object): 

    """Generate a hash for the contents of a file.""" 

    checksum = hashlib.sha1() 

    for chunk in iter(lambda: file_like_object.read(32768), b''): 

        checksum.update(chunk) 

    return checksum.hexdigest() 

 

 

@contextlib.contextmanager 

def temporary_mutation(obj, **kwargs): 

    """Temporarily change object attribute. 

 

    Temporarily set the attr on a particular object to a given value then 

    revert when finished. 

 

    One use of this is to temporarily set the read_deleted flag on a context 

    object: 

 

        with temporary_mutation(context, read_deleted="yes"): 

            do_something_that_needed_deleted_objects() 

    """ 

    def is_dict_like(thing): 

        return hasattr(thing, 'has_key') 

 

    def get(thing, attr, default): 

        if is_dict_like(thing): 

            return thing.get(attr, default) 

        else: 

            return getattr(thing, attr, default) 

 

    def set_value(thing, attr, val): 

        if is_dict_like(thing): 

            thing[attr] = val 

        else: 

            setattr(thing, attr, val) 

 

    def delete(thing, attr): 

        if is_dict_like(thing): 

            del thing[attr] 

        else: 

            delattr(thing, attr) 

 

    NOT_PRESENT = object() 

 

    old_values = {} 

    for attr, new_value in kwargs.items(): 

        old_values[attr] = get(obj, attr, NOT_PRESENT) 

        set_value(obj, attr, new_value) 

 

    try: 

        yield 

    finally: 

        for attr, old_value in old_values.items(): 

            if old_value is NOT_PRESENT: 

                delete(obj, attr) 

            else: 

                set_value(obj, attr, old_value) 

 

 

@contextlib.contextmanager 

def tempdir(**kwargs): 

    tempfile.tempdir = CONF.tempdir 

    tmpdir = tempfile.mkdtemp(**kwargs) 

    try: 

        yield tmpdir 

    finally: 

        try: 

            shutil.rmtree(tmpdir) 

        except OSError as e: 

            LOG.error(_LE('Could not remove tmpdir: %s'), e) 

 

 

def mkfs(fs, path, label=None): 

    """Format a file or block device 

 

    :param fs: Filesystem type (examples include 'swap', 'ext3', 'ext4' 

               'btrfs', etc.) 

    :param path: Path to file or block device to format 

    :param label: Volume label to use 

    """ 

    if fs == 'swap': 

        args = ['mkswap'] 

    else: 

        args = ['mkfs', '-t', fs] 

    # add -F to force no interactive execute on non-block device. 

    if fs in ('ext3', 'ext4'): 

        args.extend(['-F']) 

    if label: 

        if fs in ('msdos', 'vfat'): 

            label_opt = '-n' 

        else: 

            label_opt = '-L' 

        args.extend([label_opt, label]) 

    args.append(path) 

    try: 

        execute(*args, run_as_root=True, use_standard_locale=True) 

    except processutils.ProcessExecutionError as e: 

        with excutils.save_and_reraise_exception() as ctx: 

            if os.strerror(errno.ENOENT) in e.stderr: 

                ctx.reraise = False 

                LOG.exception(_LE('Failed to make file system. ' 

                                  'File system %s is not supported.'), fs) 

                raise exception.FileSystemNotSupported(fs=fs) 

            else: 

                LOG.exception(_LE('Failed to create a file system ' 

                                  'in %(path)s. Error: %(error)s'), 

                              {'path': path, 'error': e}) 

 

 

def unlink_without_raise(path): 

    try: 

        os.unlink(path) 

    except OSError as e: 

443        if e.errno == errno.ENOENT: 

            return 

        else: 

            LOG.warn(_LW("Failed to unlink %(path)s, error: %(e)s"), 

                        {'path': path, 'e': e}) 

 

 

def rmtree_without_raise(path): 

    try: 

        if os.path.isdir(path): 

            shutil.rmtree(path) 

    except OSError as e: 

        LOG.warn(_LW("Failed to remove dir %(path)s, error: %(e)s"), 

                {'path': path, 'e': e}) 

 

 

def write_to_file(path, contents): 

    with open(path, 'w') as f: 

        f.write(contents) 

 

 

def create_link_without_raise(source, link): 

    try: 

        os.symlink(source, link) 

    except OSError as e: 

468        if e.errno == errno.EEXIST: 

            return 

        else: 

            LOG.warn(_LW("Failed to create symlink from %(source)s to %(link)s" 

                         ", error: %(e)s"), 

                         {'source': source, 'link': link, 'e': e}) 

 

 

def safe_rstrip(value, chars=None): 

    """Removes trailing characters from a string if that does not make it empty 

 

    :param value: A string value that will be stripped. 

    :param chars: Characters to remove. 

    :return: Stripped value. 

 

    """ 

    if not isinstance(value, six.string_types): 

        LOG.warn(_LW("Failed to remove trailing character. Returning original " 

                     "object. Supplied object is not a string: %s,"), value) 

        return value 

 

    return value.rstrip(chars) or value 

 

 

def generate_uuid(): 

    return str(uuid.uuid4()) 

 

 

def is_uuid_like(val): 

    """Returns validation of a value as a UUID. 

 

    For our purposes, a UUID is a canonical form string: 

    aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa 

 

    """ 

    try: 

        return str(uuid.UUID(val)) == val 

    except (TypeError, ValueError, AttributeError): 

        return False 

 

 

def mount(src, dest, *args): 

    """Mounts a device/image file on specified location. 

 

    :param src: the path to the source file for mounting 

    :param dest: the path where it needs to be mounted. 

    :param args: a tuple containing the arguments to be 

        passed to mount command. 

    :raises: processutils.ProcessExecutionError if it failed 

        to run the process. 

    """ 

    args = ('mount', ) + args + (src, dest) 

    execute(*args, run_as_root=True, check_exit_code=[0]) 

 

 

def umount(loc, *args): 

    """Umounts a mounted location. 

 

    :param loc: the path to be unmounted. 

    :param args: a tuple containing the argumnets to be 

        passed to the umount command. 

    :raises: processutils.ProcessExecutionError if it failed 

        to run the process. 

    """ 

    args = ('umount', ) + args + (loc, ) 

    execute(*args, run_as_root=True, check_exit_code=[0]) 

 

 

def dd(src, dst, *args): 

    """Execute dd from src to dst. 

 

    :param src: the input file for dd command. 

    :param dst: the output file for dd command. 

    :param args: a tuple containing the arguments to be 

        passed to dd command. 

    :raises: processutils.ProcessExecutionError if it failed 

        to run the process. 

    """ 

    execute('dd', 'if=%s' % src, 'of=%s' % dst, *args, 

            run_as_root=True, check_exit_code=[0]) 

 

 

def is_name_safe(name): 

    """Checks whether the name is valid or not. 

 

    :param name: name of the resource. 

    :returns: True, when name is valid 

              False, otherwise. 

    """ 

    # TODO(madhuri): There should be some validation of name. 

    # Leaving it now as there is no validation 

    # while resource creation. 

    # https://bugs.launchpad.net/magnum/+bug/1430617 

    if not name: 

        return False 

    return True 

 

 

def allow_logical_names(): 

    try: 

        # v1.5 added logical name aliases 

569   573        if pecan.request.version.minor < 5: 

            return False 

    # ignore check if we're not in a pecan context 

    except AttributeError: 

        pass 

    return True 

 

 

def raise_exception_invalid_scheme(url): 

    valid_schemes = ['http', 'https'] 

 

    if not isinstance(url, basestring): 

        raise exception.Urllib2InvalidScheme(url=url) 

 

    scheme = url.split(':')[0] 

    if scheme not in valid_schemes: 

        raise exception.Urllib2InvalidScheme(url=url)