diff --git a/.gitignore b/.gitignore index 78aa674d32..3bcf547dd9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ pyshtables.py /bin/bitbakec *.swp tags +.coverage +htmlcov/ diff --git a/bin/bitbake-selftest b/bin/bitbake-selftest index 48a58fef67..6307613d3c 100755 --- a/bin/bitbake-selftest +++ b/bin/bitbake-selftest @@ -15,24 +15,36 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import os -import sys, logging -sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) +""" + +Prerequirements: +% pip install nose +% pip install coverage + +* bitbake-selftest Usage: + + * Code coverage analysis -import unittest -try: - import bb -except RuntimeError as exc: - sys.exit(str(exc)) + Run all tests with coverage from bitbake top dir: + % ./bin/bitbake-selftest --with-coverage -tests = ["bb.tests.codeparser", - "bb.tests.cow", - "bb.tests.data", - "bb.tests.fetch", - "bb.tests.utils"] + Generate HTML coverage reports under htmlcov (default): + % coverage html && firefox htmlcov/index.html -for t in tests: - __import__(t) + * Has same options as nosetests -unittest.main(argv=["bitbake-selftest"] + tests) + Specify a specific unit testsuite: + % ./bin/bitbake-selftest -v lib/bb/tests/test_data.py + + Specify a specific testcase from a test suite: + % ./bin/bitbake-selftest -v lib/bb/tests/test_fetch.py:FetcherNetworkTest.test_gitfetch + +""" + +import os +import sys +import nose +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + '/lib') +if __name__ == '__main__': + nose.run(argv=sys.argv) diff --git a/lib/bb/COW.py b/lib/bb/COW.py index 6917ec378a..dad03b3ed6 100644 --- a/lib/bb/COW.py +++ b/lib/bb/COW.py @@ -18,7 +18,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -#Please Note: +# Please Note: # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW. # Assign a file to __warn__ to get warnings about slow operations. # @@ -40,46 +40,63 @@ MUTABLE = "__mutable__" +IGNORELIST = ['__module__', '__doc__', # Python's default builtins + '__count__', # from class COWDictBase, COWSetBase + '__hasmutable__', # from COWDictMeta + ] + + class COWMeta(type): pass + class COWDictMeta(COWMeta): - __warn__ = False + __warn__ = None __hasmutable__ = False __marker__ = tuple() - def __str__(cls): - # FIXME: I have magic numbers! - return "" % (cls.__count__, len(cls.__dict__) - 3) - __repr__ = __str__ - - def cow(cls): - class C(cls): + def copy(cls): + class COWDict(cls): __count__ = cls.__count__ + 1 - return C - copy = cow - __call__ = cow + return COWDict + + __call__ = copy + + def count(cls): + return cls.__count__ def __setitem__(cls, key, value): + if not isinstance(key, str): + raise TypeError("%s: user key must be a string" % cls.__name__) + if key.startswith('__'): + # It does not make sense to let the user enter keys starting with + # '__' since we risk to overwrite existing Python builtins or + # even our own builtins + raise TypeError("%s: user key is not allowed to start with '__'" % + cls.__name__) if not isinstance(value, ImmutableTypes): if not isinstance(value, COWMeta): cls.__hasmutable__ = True - key += MUTABLE + key += MUTABLE # mutable keys will be suffixed by "__mutable__" setattr(cls, key, value) def __getmutable__(cls, key, readonly=False): + # Add the __mutable__ suffix to the key nkey = key + MUTABLE try: return cls.__dict__[nkey] except KeyError: pass + # Get nkey's value otherwise will raise AttributeError value = getattr(cls, nkey) if readonly: return value - if not cls.__warn__ is False and not isinstance(value, COWMeta): - print("Warning: Doing a copy because %s is a mutable type." % key, file=cls.__warn__) + if cls.__warn__ and not isinstance(value, COWMeta): + print("Warning: Doing a copy because %s is a mutable type." % + key, file=cls.__warn__) + try: value = value.copy() except AttributeError as e: @@ -88,17 +105,21 @@ def __getmutable__(cls, key, readonly=False): return value __getmarker__ = [] + def __getreadonly__(cls, key, default=__getmarker__): - """\ + """ Get a value (even if mutable) which you promise not to change. """ return cls.__getitem__(key, default, True) def __getitem__(cls, key, default=__getmarker__, readonly=False): + """ This method is called when calling obj[key] """ try: try: + # Check if the key is present in the attribute list value = getattr(cls, key) except AttributeError: + # Check if the key is mutable (with '__mutable__' suffix) value = cls.__getmutable__(key, readonly) # This is for values which have been deleted @@ -129,15 +150,17 @@ def has_key(cls, key): return False return True - def iter(cls, type, readonly=False): + def iter(cls, type_str, readonly=False): for key in dir(cls): + # We skip Python's builtins and the ones in IGNORELIST if key.startswith("__"): continue + # Mutable keys have a __mutable__ suffix that we remove if key.endswith(MUTABLE): key = key[:-len(MUTABLE)] - if type == "keys": + if type_str == "keys": yield key try: @@ -148,33 +171,58 @@ def iter(cls, type, readonly=False): except KeyError: continue - if type == "values": + if type_str == "values": yield value - if type == "items": + if type_str == "items": yield (key, value) raise StopIteration() - def iterkeys(cls): - return cls.iter("keys") + def iterkeys(cls, readonly=False): + return cls.iter("keys", readonly) + + # The default iterator is 'readonly' + def __iter__(cls): + return cls.iter("keys", readonly=True) + def itervalues(cls, readonly=False): - if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) + if cls.__warn__ and cls.__hasmutable__ and readonly is False: + print( + "Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) return cls.iter("values", readonly) + def iteritems(cls, readonly=False): - if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) + if cls.__warn__ and cls.__hasmutable__ and readonly is False: + print( + "Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) return cls.iter("items", readonly) -class COWSetMeta(COWDictMeta): def __str__(cls): - # FIXME: I have magic numbers! - return "" % (cls.__count__, len(cls.__dict__) -3) + """ + Returns a string representation of this object + The number of keys is only showing keys in the current 'level' + """ + return ("<%s Level: %i Number of keys: %i>" % + (cls.__name__, cls.__count__, cls.__len__())) __repr__ = __str__ - def cow(cls): - class C(cls): + def __len__(cls): + """ Returns the number of 'keys' in the COWDict """ + # cls.__dict__ is the default module namespace as a dictionary + # We skip keys found in IGNORELIST + i = 0 + for x in cls.__dict__: + if x in IGNORELIST: + continue + i += 1 + return i + + +class COWSetMeta(COWDictMeta): + + def copy(cls): + class COWSet(cls): __count__ = cls.__count__ + 1 - return C + return COWSet def add(cls, value): COWDictMeta.__setitem__(cls, repr(hash(value)), value) @@ -182,8 +230,8 @@ def add(cls, value): def remove(cls, value): COWDictMeta.__delitem__(cls, repr(hash(value))) - def __in__(cls, value): - return COWDictMeta.has_key(repr(hash(value))) + def __contains__(cls, value): + return COWDictMeta.has_key(cls, repr(hash(value))) def iterkeys(cls): raise TypeError("sets don't have keys") @@ -191,133 +239,17 @@ def iterkeys(cls): def iteritems(cls): raise TypeError("sets don't have 'items'") + # These are the actual classes you use! class COWDictBase(object): __metaclass__ = COWDictMeta __count__ = 0 + class COWSetBase(object): __metaclass__ = COWSetMeta __count__ = 0 + if __name__ == "__main__": - import sys - COWDictBase.__warn__ = sys.stderr - a = COWDictBase() - print("a", a) - - a['a'] = 'a' - a['b'] = 'b' - a['dict'] = {} - - b = a.copy() - print("b", b) - b['c'] = 'b' - - print() - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(): - print(x) - print() - - b['dict']['a'] = 'b' - b['a'] = 'c' - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(): - print(x) - print() - - try: - b['dict2'] - except KeyError as e: - print("Okay!") - - a['set'] = COWSetBase() - a['set'].add("o1") - a['set'].add("o1") - a['set'].add("o2") - - print("a", a) - for x in a['set'].itervalues(): - print(x) - print("--") - print("b", b) - for x in b['set'].itervalues(): - print(x) - print() - - b['set'].add('o3') - - print("a", a) - for x in a['set'].itervalues(): - print(x) - print("--") - print("b", b) - for x in b['set'].itervalues(): - print(x) - print() - - a['set2'] = set() - a['set2'].add("o1") - a['set2'].add("o1") - a['set2'].add("o2") - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - del b['b'] - try: - print(b['b']) - except KeyError: - print("Yay! deleted key raises error") - - if b.has_key('b'): - print("Boo!") - else: - print("Yay - has_key with delete works!") - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - b.__revertitem__('b') - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - b.__revertitem__('dict') - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() + print("The unit tests in test_cow.py show how COWDict/SetBase are used") diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py index c0c7735637..98f3e29c20 100644 --- a/lib/bb/__init__.py +++ b/lib/bb/__init__.py @@ -21,7 +21,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -__version__ = "1.21.0" +__version__ = "1.21.0-p1" import sys if sys.version_info < (2, 7, 3): @@ -102,42 +102,7 @@ def fatal(*args): sys.exit(1) -def deprecated(func, name=None, advice=""): - """This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emmitted - when the function is used.""" - import warnings - - if advice: - advice = ": %s" % advice - if name is None: - name = func.__name__ - - def newFunc(*args, **kwargs): - warnings.warn("Call to deprecated function %s%s." % (name, - advice), - category=DeprecationWarning, - stacklevel=2) - return func(*args, **kwargs) - newFunc.__name__ = func.__name__ - newFunc.__doc__ = func.__doc__ - newFunc.__dict__.update(func.__dict__) - return newFunc - -# For compatibility -def deprecate_import(current, modulename, fromlist, renames = None): - """Import objects from one module into another, wrapping them with a DeprecationWarning""" - import sys - - module = __import__(modulename, fromlist = fromlist) - for position, objname in enumerate(fromlist): - obj = getattr(module, objname) - newobj = deprecated(obj, "{0}.{1}".format(current, objname), - "Please use {0}.{1} instead".format(modulename, objname)) - if renames: - newname = renames[position] - else: - newname = objname - - setattr(sys.modules[current], newname, newobj) + + + diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index a1cbaba62b..6b66cfc441 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -28,22 +28,29 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # Based on functions from the base bb module, Copyright 2003 Holger Schurig -import copy, re, sys, traceback +import copy +import re +import sys +import traceback from collections import MutableMapping import logging import hashlib -import bb, bb.codeparser -from bb import utils -from bb.COW import COWDictBase +import bb +import bb.codeparser +import bb.parse +from bb import utils +from bb.COW import COWDictBase logger = logging.getLogger("BitBake.Data") __setvar_keyword__ = ["_append", "_prepend", "_remove"] -__setvar_regexp__ = re.compile('(?P.*?)(?P_append|_prepend|_remove)(_(?P.*))?$') +__setvar_regexp__ = re.compile( + '(?P.*?)(?P_append|_prepend|_remove)(_(?P.*))?$') __expand_var_regexp__ = re.compile(r"\${[^{}@\n\t ]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") -def infer_caller_details(loginfo, parent = False, varval = True): + +def infer_caller_details(loginfo, parent=False, varval=True): """Save the caller the trouble of specifying everything.""" # Save effort. if 'ignore' in loginfo and loginfo['ignore']: @@ -52,7 +59,7 @@ def infer_caller_details(loginfo, parent = False, varval = True): if not loginfo: loginfo['ignore'] = True return - # Infer caller's likely values for variable (var) and value (value), + # Infer caller's likely values for variable (var) and value (value), # to reduce clutter in the rest of the code. if varval and ('variable' not in loginfo or 'detail' not in loginfo): try: @@ -71,17 +78,19 @@ def infer_caller_details(loginfo, parent = False, varval = True): loginfo['variable'] = v # Infer file/line/function from traceback if 'file' not in loginfo: - depth = 3 + depth = 3 if parent: depth = 4 - file, line, func, text = traceback.extract_stack(limit = depth)[0] - loginfo['file'] = file + file_, line, func, text = traceback.extract_stack(limit=depth)[0] + loginfo['file'] = file_ loginfo['line'] = line if func not in loginfo: loginfo['func'] = func + class VariableParse: - def __init__(self, varname, d, val = None): + + def __init__(self, varname, d, val=None): self.varname = varname self.d = d self.value = val @@ -90,41 +99,53 @@ def __init__(self, varname, d, val = None): self.execs = set() def var_sub(self, match): - key = match.group()[2:-1] - if self.varname and key: - if self.varname == key: - raise Exception("variable %s references itself!" % self.varname) - if key in self.d.expand_cache: - varparse = self.d.expand_cache[key] - var = varparse.value - else: - var = self.d.getVar(key, True) - self.references.add(key) - if var is not None: - return var - else: - return match.group() + key = match.group()[2:-1] + if self.varname and key: + if self.varname == key: + raise Exception( + "variable %s references itself!" % self.varname) + if key in self.d.expand_cache: + varparse = self.d.expand_cache[key] + var = varparse.value + else: + var = self.d.getVar(key, True) + self.references.add(key) + if var is not None: + return var + else: + return match.group() def python_sub(self, match): - code = match.group()[3:-1] - codeobj = compile(code.strip(), self.varname or "", "eval") - - parser = bb.codeparser.PythonParser(self.varname, logger) - parser.parse_python(code) - if self.varname: - vardeps = self.d.getVarFlag(self.varname, "vardeps", True) - if vardeps is None: - parser.log.flush() - else: + code = match.group()[3:-1] + # compile builtin: compile(source, filename, mode[, flags[, dont_inherit]]) + # Compile the source into a code or AST object + # The filename argument is the file name from which the code was read; + # pass some recognizable value if it wasn't read from a file: '' + # The mode argument specifies what kind of code must be compiled + # ('eval' if it consists of a single expression) + # Use when to be able to run with nose option '--with-coverage' + codeobj = compile(code.strip(), "", "eval") + + parser = bb.codeparser.PythonParser(self.varname, logger) + parser.parse_python(code) + if self.varname: + vardeps = self.d.getVarFlag(self.varname, "vardeps", True) + if vardeps is None: parser.log.flush() - self.references |= parser.references - self.execs |= parser.execs + else: + parser.log.flush() + self.references |= parser.references + self.execs |= parser.execs - value = utils.better_eval(codeobj, DataContext(self.d)) - return str(value) + value = utils.better_eval(codeobj, DataContext(self.d)) + return str(value) + + def __str__(self): + return str(self.varname) + " : " + str(self.value) class DataContext(dict): + def __init__(self, metadata, **kwargs): self.metadata = metadata dict.__init__(self, **kwargs) @@ -137,25 +158,33 @@ def __missing__(self, key): else: return value + class ExpansionError(Exception): + def __init__(self, varname, expression, exception): self.expression = expression self.variablename = varname self.exception = exception if varname: if expression: - self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) + self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % ( + varname, expression, type(exception).__name__, exception) else: - self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) + self.msg = "Failure expanding variable %s: %s: %s" % ( + varname, type(exception).__name__, exception) else: - self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) + self.msg = "Failure expanding expression %s which triggered exception %s: %s" % ( + expression, type(exception).__name__, exception) Exception.__init__(self, self.msg) self.args = (varname, expression, exception) + def __str__(self): return self.msg + class IncludeHistory(object): - def __init__(self, parent = None, filename = '[TOP LEVEL]'): + + def __init__(self, parent=None, filename='[TOP LEVEL]'): self.parent = parent self.filename = filename self.children = [] @@ -180,10 +209,11 @@ def __exit__(self, a, b, c): if self.current.parent: self.current = self.current.parent else: - bb.warn("Include log: Tried to finish '%s' at top level." % filename) + bb.warn("Include log: Tried to finish '%s' at top level." % + filename) return False - def emit(self, o, level = 0): + def emit(self, o, level=0): """Emit an include history file, and its children.""" if level: spaces = " " * (level - 1) @@ -197,10 +227,12 @@ def emit(self, o, level = 0): o.write("\n") child.emit(o, level) + class VariableHistory(object): + def __init__(self, dataroot): self.dataroot = dataroot - self.variables = COWDictBase.copy() + self.variables = COWDictBase() def copy(self): new = VariableHistory(self.dataroot) @@ -212,7 +244,7 @@ def record(self, *kwonly, **loginfo): return if len(kwonly) > 0: raise TypeError - infer_caller_details(loginfo, parent = True) + infer_caller_details(loginfo, parent=True) if 'ignore' in loginfo and loginfo['ignore']: return if 'op' not in loginfo or not loginfo['op']: @@ -244,9 +276,9 @@ def emit(self, var, oval, val, o): for event in history: # o.write("# %s\n" % str(event)) if 'func' in event: - # If we have a function listed, this is internal - # code, not an operation in a config file, and the - # full path is distracting. + # If we have a function listed, this is internal + # code, not an operation in a config file, and the + # full path is distracting. event['file'] = re.sub('.*/', '', event['file']) display_func = ' [%s]' % event['func'] else: @@ -255,7 +287,10 @@ def emit(self, var, oval, val, o): flag = '[%s] ' % (event['flag']) else: flag = '' - o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % + (event['op'], event['file'], event['line'], + display_func, flag, re.sub('\n', '\n# ', + event['detail']))) if len(history) > 1: o.write("# computed:\n") o.write('# "%s"\n' % (commentVal)) @@ -276,7 +311,7 @@ def get_variable_lines(self, var, f): var_history = self.variable(var) lines = [] for event in var_history: - if f== event['file']: + if f == event['file']: line = event['line'] lines.append(line) return lines @@ -285,12 +320,14 @@ def del_var_history(self, var, f=None, line=None): """If file f and line are not given, the entire history of var is deleted""" if var in self.variables: if f and line: - self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] + self.variables[var] = [x for x in self.variables[var] if x['file'] != f and x['line'] != line] else: self.variables[var] = [] + class DataSmart(MutableMapping): - def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ): + + def __init__(self, special=None, seen=None): self.dict = {} self.inchistory = IncludeHistory() @@ -298,8 +335,15 @@ def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ): self._tracking = False # cookie monster tribute - self._special_values = special - self._seen_overrides = seen + if special is None: + self._special_values = COWDictBase() + else: + self._special_values = special + + if seen is None: + self._seen_overrides = COWDictBase() + else: + self._seen_overrides = seen self.expand_cache = {} @@ -311,7 +355,7 @@ def disableTracking(self): def expandWithRefs(self, s, varname): - if not isinstance(s, basestring): # sanity check + if not isinstance(s, basestring): # sanity check return VariableParse(varname, self, s) if varname and varname in self.expand_cache: @@ -322,7 +366,9 @@ def expandWithRefs(self, s, varname): while s.find('${') != -1: olds = s try: + # Variable substitution: ${FOO} s = __expand_var_regexp__.sub(varparse.var_sub, s) + # Python expression substitution: ${@} s = __expand_python_regexp__.sub(varparse.python_sub, s) if s == olds: break @@ -340,18 +386,17 @@ def expandWithRefs(self, s, varname): return varparse - def expand(self, s, varname = None): + def expand(self, s, varname=None): return self.expandWithRefs(s, varname).value - - def finalize(self, parent = False): + def finalize(self, parent=False): """Performs final steps upon the datastore, including application of overrides""" overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] finalize_caller = { 'op': 'finalize', } - infer_caller_details(finalize_caller, parent = parent, varval = False) + infer_caller_details(finalize_caller, parent=parent, varval=False) # # Well let us see what breaks here. We used to iterate @@ -380,15 +425,16 @@ def finalize(self, parent = False): if o not in self._seen_overrides: continue - vars = self._seen_overrides[o].copy() - for var in vars: + vars_ = self._seen_overrides[o].copy() + for var in vars_: name = var[:-l] try: # Report only once, even if multiple changes. if name not in finalizes_reported: finalizes_reported[name] = True finalize_caller['variable'] = name - finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False)) + finalize_caller['detail'] = 'was: ' + str( + self.getVar(name, False)) self.varhistory.record(**finalize_caller) # Copy history of the override over. for event in self.varhistory.variable(var): @@ -396,7 +442,8 @@ def finalize(self, parent = False): loginfo['variable'] = name loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op']) self.varhistory.record(**loginfo) - self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '') + self.setVar( + name, self.getVar(var, False), op='finalize', file='override[%s]' % o, line='') self.delVar(var) except Exception: logger.info("Untracked delVar") @@ -414,7 +461,7 @@ def finalize(self, parent = False): if not o2 in overrides: match = False if not match: - keep.append((a ,o)) + keep.append((a, o)) continue if op == "_append": @@ -425,9 +472,11 @@ def finalize(self, parent = False): sval = a + (self.getVar(append, False) or "") self.setVar(append, sval) elif op == "_remove": - removes = self.getVarFlag(append, "_removeactive", False) or [] + removes = self.getVarFlag( + append, "_removeactive", False) or [] removes.extend(a.split()) - self.setVarFlag(append, "_removeactive", removes, ignore=True) + self.setVarFlag( + append, "_removeactive", removes, ignore=True) # We save overrides that may be applied at some later stage if keep: @@ -461,13 +510,12 @@ def _makeShadowCopy(self, var): else: self.initVar(var) - def setVar(self, var, value, **loginfo): - #print("var=" + str(var) + " val=" + str(value)) + # print("var=" + str(var) + " val=" + str(value)) if 'op' not in loginfo: loginfo['op'] = "set" self.expand_cache = {} - match = __setvar_regexp__.match(var) + match = __setvar_regexp__.match(var) if match and match.group("keyword") in __setvar_keyword__: base = match.group('base') keyword = match.group("keyword") @@ -506,11 +554,11 @@ def setVar(self, var, value, **loginfo): def _setvar_update_overrides(self, var): # aka pay the cookie monster - override = var[var.rfind('_')+1:] + override = var[var.rfind('_') + 1:] if len(override) > 0: if override not in self._seen_overrides: self._seen_overrides[override] = set() - self._seen_overrides[override].add( var ) + self._seen_overrides[override].add(var) def getVar(self, var, expand=False, noweakdefault=False): return self.getVarFlag(var, "_content", expand, noweakdefault) @@ -565,7 +613,7 @@ def delVar(self, var, **loginfo): self.expand_cache = {} self.dict[var] = {} if '_' in var: - override = var[var.rfind('_')+1:] + override = var[var.rfind('_') + 1:] if override and override in self._seen_overrides and var in self._seen_overrides[override]: self._seen_overrides[override].remove(var) @@ -652,7 +700,7 @@ def setVarFlags(self, var, flags, **loginfo): self.varhistory.record(**loginfo) self.dict[var][i] = flags[i] - def getVarFlags(self, var, expand = False, internalflags=False): + def getVarFlags(self, var, expand=False, internalflags=False): local_var = self._findVar(var) flags = {} @@ -667,7 +715,6 @@ def getVarFlags(self, var, expand = False, internalflags=False): return None return flags - def delVarFlags(self, var, **loginfo): if not var in self.dict: self._makeShadowCopy(var) @@ -680,19 +727,19 @@ def delVarFlags(self, var, **loginfo): # try to save the content if "_content" in self.dict[var]: - content = self.dict[var]["_content"] - self.dict[var] = {} + content = self.dict[var]["_content"] + self.dict[var] = {} self.dict[var]["_content"] = content else: del self.dict[var] - def createCopy(self): """ Create a copy of self by setting _data to self """ # we really want this to be a DataSmart... - data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) + data = DataSmart( + seen=self._seen_overrides.copy(), special=self._special_values.copy()) data.dict["_data"] = self.dict data.varhistory = self.varhistory.copy() data.varhistory.datasmart = data @@ -724,7 +771,7 @@ def localkeys(self): yield key def __iter__(self): - def keylist(d): + def keylist(d): klist = set() for key in d: if key == "_data": @@ -739,7 +786,7 @@ def keylist(d): return klist for k in keylist(self.dict): - yield k + yield k def __len__(self): return len(frozenset(self)) @@ -763,32 +810,33 @@ def get_hash(self): bb.data.expandKeys(d) bb.data.update_data(d) - config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) + config_whitelist = set( + (d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) keys = set(key for key in iter(d) if not key.startswith("__")) for key in keys: if key in config_whitelist: continue value = d.getVar(key, False) or "" - data.update({key:value}) + data.update({key: value}) - varflags = d.getVarFlags(key, internalflags = True) + varflags = d.getVarFlags(key, internalflags=True) if not varflags: continue for f in varflags: if f == "_content": continue - data.update({'%s[%s]' % (key, f):varflags[f]}) + data.update({'%s[%s]' % (key, f): varflags[f]}) for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: bb_list = d.getVar(key, False) or [] bb_list.sort() - data.update({key:str(bb_list)}) + data.update({key: str(bb_list)}) if key == "__BBANONFUNCS": for i in bb_list: value = d.getVar(i, True) or "" - data.update({i:value}) + data.update({i: value}) data_str = str([(k, data[k]) for k in sorted(data.keys())]) return hashlib.md5(data_str).hexdigest() diff --git a/lib/bb/parse/parse_py/BBHandler.py b/lib/bb/parse/parse_py/BBHandler.py index 01f22d3b24..acc7c1b391 100644 --- a/lib/bb/parse/parse_py/BBHandler.py +++ b/lib/bb/parse/parse_py/BBHandler.py @@ -28,15 +28,58 @@ class for handling .bb files from __future__ import absolute_import import re, bb, os import logging +#from bb import deprecate_import import bb.build, bb.utils -from bb import data +import bb.data +#from bb import data from . import ConfHandler from .. import resolve_file, ast, logger from .ConfHandler import include, init +# I had to move deprecated and deprecate_import here to avoid an import error +# and this is actually only used in this module +def deprecated(func, name=None, advice=""): + """This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emmitted + when the function is used.""" + import warnings + + if advice: + advice = ": %s" % advice + if name is None: + name = func.__name__ + + def newFunc(*args, **kwargs): + warnings.warn("Call to deprecated function %s%s." % (name, + advice), + category=DeprecationWarning, + stacklevel=2) + return func(*args, **kwargs) + newFunc.__name__ = func.__name__ + newFunc.__doc__ = func.__doc__ + newFunc.__dict__.update(func.__dict__) + return newFunc + +# For compatibility +def deprecate_import(current, modulename, fromlist, renames = None): + """Import objects from one module into another, wrapping them with a DeprecationWarning""" + import sys + + module = __import__(modulename, fromlist = fromlist) + for position, objname in enumerate(fromlist): + obj = getattr(module, objname) + newobj = deprecated(obj, "{0}.{1}".format(current, objname), + "Please use {0}.{1} instead".format(modulename, objname)) + if renames: + newname = renames[position] + else: + newname = objname + + setattr(sys.modules[current], newname, newobj) + # For compatibility -bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"]) +deprecate_import(__name__, "bb.parse", ["vars_from_file"]) __func_start_regexp__ = re.compile( r"(((?Ppython)|(?Pfakeroot))\s*)*(?P[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" ) __inherit_regexp__ = re.compile( r"inherit\s+(.+)" ) diff --git a/lib/bb/parse/parse_py/__init__.py b/lib/bb/parse/parse_py/__init__.py index 3e658d0de9..75ca47c717 100644 --- a/lib/bb/parse/parse_py/__init__.py +++ b/lib/bb/parse/parse_py/__init__.py @@ -27,7 +27,7 @@ # Based on functions from the base bb module, Copyright 2003 Holger Schurig from __future__ import absolute_import -from . import ConfHandler -from . import BBHandler +from bb.parse.parse_py import ConfHandler +from bb.parse.parse_py import BBHandler __version__ = '1.0' diff --git a/lib/bb/tests/cow.py b/lib/bb/tests/cow.py deleted file mode 100644 index 35c5841f32..0000000000 --- a/lib/bb/tests/cow.py +++ /dev/null @@ -1,136 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake Tests for Copy-on-Write (cow.py) -# -# Copyright 2006 Holger Freyther -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -import unittest -import os - -class COWTestCase(unittest.TestCase): - """ - Test case for the COW module from mithro - """ - - def testGetSet(self): - """ - Test and set - """ - from bb.COW import COWDictBase - a = COWDictBase.copy() - - self.assertEquals(False, a.has_key('a')) - - a['a'] = 'a' - a['b'] = 'b' - self.assertEquals(True, a.has_key('a')) - self.assertEquals(True, a.has_key('b')) - self.assertEquals('a', a['a'] ) - self.assertEquals('b', a['b'] ) - - def testCopyCopy(self): - """ - Test the copy of copies - """ - - from bb.COW import COWDictBase - - # create two COW dict 'instances' - b = COWDictBase.copy() - c = COWDictBase.copy() - - # assign some keys to one instance, some keys to another - b['a'] = 10 - b['c'] = 20 - c['a'] = 30 - - # test separation of the two instances - self.assertEquals(False, c.has_key('c')) - self.assertEquals(30, c['a']) - self.assertEquals(10, b['a']) - - # test copy - b_2 = b.copy() - c_2 = c.copy() - - self.assertEquals(False, c_2.has_key('c')) - self.assertEquals(10, b_2['a']) - - b_2['d'] = 40 - self.assertEquals(False, c_2.has_key('d')) - self.assertEquals(True, b_2.has_key('d')) - self.assertEquals(40, b_2['d']) - self.assertEquals(False, b.has_key('d')) - self.assertEquals(False, c.has_key('d')) - - c_2['d'] = 30 - self.assertEquals(True, c_2.has_key('d')) - self.assertEquals(True, b_2.has_key('d')) - self.assertEquals(30, c_2['d']) - self.assertEquals(40, b_2['d']) - self.assertEquals(False, b.has_key('d')) - self.assertEquals(False, c.has_key('d')) - - # test copy of the copy - c_3 = c_2.copy() - b_3 = b_2.copy() - b_3_2 = b_2.copy() - - c_3['e'] = 4711 - self.assertEquals(4711, c_3['e']) - self.assertEquals(False, c_2.has_key('e')) - self.assertEquals(False, b_3.has_key('e')) - self.assertEquals(False, b_3_2.has_key('e')) - self.assertEquals(False, b_2.has_key('e')) - - b_3['e'] = 'viel' - self.assertEquals('viel', b_3['e']) - self.assertEquals(4711, c_3['e']) - self.assertEquals(False, c_2.has_key('e')) - self.assertEquals(True, b_3.has_key('e')) - self.assertEquals(False, b_3_2.has_key('e')) - self.assertEquals(False, b_2.has_key('e')) - - def testCow(self): - from bb.COW import COWDictBase - c = COWDictBase.copy() - c['123'] = 1027 - c['other'] = 4711 - c['d'] = { 'abc' : 10, 'bcd' : 20 } - - copy = c.copy() - - self.assertEquals(1027, c['123']) - self.assertEquals(4711, c['other']) - self.assertEquals({'abc':10, 'bcd':20}, c['d']) - self.assertEquals(1027, copy['123']) - self.assertEquals(4711, copy['other']) - self.assertEquals({'abc':10, 'bcd':20}, copy['d']) - - # cow it now - copy['123'] = 1028 - copy['other'] = 4712 - copy['d']['abc'] = 20 - - - self.assertEquals(1027, c['123']) - self.assertEquals(4711, c['other']) - self.assertEquals({'abc':10, 'bcd':20}, c['d']) - self.assertEquals(1028, copy['123']) - self.assertEquals(4712, copy['other']) - self.assertEquals({'abc':20, 'bcd':20}, copy['d']) diff --git a/lib/bb/tests/codeparser.py b/lib/bb/tests/test_codeparser.py similarity index 100% rename from lib/bb/tests/codeparser.py rename to lib/bb/tests/test_codeparser.py diff --git a/lib/bb/tests/test_cow.py b/lib/bb/tests/test_cow.py new file mode 100644 index 0000000000..140f70dba5 --- /dev/null +++ b/lib/bb/tests/test_cow.py @@ -0,0 +1,319 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Tests for Copy-on-Write (cow.py) +# +# Copyright 2006 Holger Freyther +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import os +from bb.COW import COWDictBase, COWSetBase +from nose.tools import raises + + +class TestCOWDictBase(unittest.TestCase): + + """ + Test case for the COW module from mithro + """ + + def test_cow(self): + # Test base COW functionality + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + + self.assertEquals(c.count(), 1) # Level: 1 + self.assertEquals(len(c), 3) # c has 3 keys in Level #1 + + c_2 = c.copy() + + self.assertEquals(c_2.count(), 2) # c_2 Level: 2 (copy of c) + self.assertEquals(len(c_2), 0) # c_2 has 0 keys in Level #2 + self.assertEquals(1027, c['123']) + self.assertEquals(4711, c['other']) + self.assertEquals(d, c['d']) + self.assertEquals(1027, c_2['123']) + self.assertEquals(4711, c_2['other']) + + # The two dictionary objects must be identical at this point: + # We must use getattr() instead of c_2['d'] + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + # c_2['d'] will result in calling __getmutable__ and since 'd__mutable__' + # is not an attribute of c_2 (but only of c), and since readonly=False, + # it will do a d.copy() aka a 'shallow copy' of the dictionary 'd' + # So the copy on write behaviour actually happens here: + self.assertEquals(d, c_2['d']) + + # At this point, the two dictionary objects must be different: + # We must use getattr() instead of c_2['d'] + self.assertNotEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + # At this point c_2 has 1 key in Level #2 + self.assertEquals(c_2.count(), 2) + self.assertEquals(len(c_2), 1) + + # Change the immutable values for the same keys as in c + c_2['123'] = 1028 + c_2['other'] = 4712 + + # Since we only changed c_2, verify again that that c is unchanged + self.assertEquals(1027, c['123']) + self.assertEquals(4711, c['other']) + # However, c_2 values must have changed + self.assertEquals(1028, c_2['123']) + self.assertEquals(4712, c_2['other']) + + # Change the mutable values for the same keys as in c + c_2['d']['abc'] = 20 + + # Since we only changed c_2, verify again that that c is unchanged + self.assertEquals(d, c['d']) + # However, c_2 is changed + self.assertEquals({'abc': 20, 'bcd': 20}, c_2['d']) + + # At this point c_2 has 3 keys in Level #2 + self.assertEquals(c_2.count(), 2) + self.assertEquals(len(c_2), 3) + + def test_iter_readonly(self): + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + expected_keys = ('123', 'other', 'd') + expected_values = (1027, 4711, d) + expected_items = (('123', 1027), ('other', 4711), ('d', d)) + + i = 0 + for x in c.iterkeys(readonly=True): + i += 1 + self.assertTrue(x in expected_keys) + self.assertTrue(i == 3) + + i = 0 + for x in c.itervalues(readonly=True): + i += 1 + self.assertTrue(x in expected_values) + self.assertTrue(i == 3) + + i = 0 + for x in c.iteritems(readonly=True): + i += 1 + self.assertTrue(x in expected_items) + self.assertTrue(i == 3) + + c_2 = c.copy() + + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + i = 0 + for x in c_2.iterkeys(readonly=True): + i += 1 + self.assertTrue(x in expected_keys) + self.assertTrue(i == 3) + + # Check that the mutable dict 'd' has not been shallow copied + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + i = 0 + for x in c_2.itervalues(readonly=True): + i += 1 + self.assertTrue(x in expected_values) + self.assertTrue(i == 3) + + i = 0 + for x in c.iteritems(readonly=True): + i += 1 + self.assertTrue(x in expected_items) + self.assertTrue(i == 3) + + def test_default_ro_iter(self): + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + expected_keys = ('123', 'other', 'd') + + c_2 = c.copy() + + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + i = 0 + for x in c: + i += 1 + self.assertTrue(x in expected_keys) + self.assertTrue(i == 3) + + # Check that the mutable dict 'd' has not been shallow copied + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + def test_nonro_iteritems(self): + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + expected_keys = ('123', 'other', 'd') + + c_2 = c.copy() + + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + i = 0 + for k, v in c_2.iteritems(): + i += 1 + self.assertTrue(k in expected_keys) + self.assertTrue(i == 3) + + # Check that the mutable dict 'd' _has been_ shallow copied + self.assertNotEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + def test_cow_get_set(self): + a = COWDictBase() + self.assertEquals(False, a.has_key('a')) + + a['a'] = 'a' + a['b'] = 'b' + self.assertEquals(True, a.has_key('a')) + self.assertEquals(True, a.has_key('b')) + self.assertEquals('a', a['a']) + self.assertEquals('b', a['b']) + + def test_cow_copy_of_copy(self): + # Test the copy of copies + + # create two COW dict 'instances' + b = COWDictBase() + c = COWDictBase() + + # assign some keys to one instance, some keys to another + b['a'] = 10 + b['c'] = 20 + c['a'] = 30 + + # test separation of the two instances + self.assertEquals(False, c.has_key('c')) + self.assertEquals(30, c['a']) + self.assertEquals(10, b['a']) + + # test copy + b_2 = b.copy() + c_2 = c.copy() + + self.assertEquals(False, c_2.has_key('c')) + self.assertEquals(10, b_2['a']) + + b_2['d'] = 40 + self.assertEquals(False, c_2.has_key('d')) + self.assertEquals(True, b_2.has_key('d')) + self.assertEquals(40, b_2['d']) + self.assertEquals(False, b.has_key('d')) + self.assertEquals(False, c.has_key('d')) + + c_2['d'] = 30 + self.assertEquals(True, c_2.has_key('d')) + self.assertEquals(True, b_2.has_key('d')) + self.assertEquals(30, c_2['d']) + self.assertEquals(40, b_2['d']) + self.assertEquals(False, b.has_key('d')) + self.assertEquals(False, c.has_key('d')) + + # test copy of the copy + c_3 = c_2.copy() + b_3 = b_2.copy() + b_3_2 = b_2.copy() + + c_3['e'] = 4711 + self.assertEquals(4711, c_3['e']) + self.assertEquals(False, c_2.has_key('e')) + self.assertEquals(False, b_3.has_key('e')) + self.assertEquals(False, b_3_2.has_key('e')) + self.assertEquals(False, b_2.has_key('e')) + + b_3['e'] = 'viel' + self.assertEquals('viel', b_3['e']) + self.assertEquals(4711, c_3['e']) + self.assertEquals(False, c_2.has_key('e')) + self.assertEquals(True, b_3.has_key('e')) + self.assertEquals(False, b_3_2.has_key('e')) + self.assertEquals(False, b_2.has_key('e')) + + def test_contains_ro(self): + c = COWDictBase() + c['xyz'] = 1 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + self.assertTrue('xyz' in c) + c_2 = c.copy() + self.assertTrue('d' in c) + self.assertTrue('d' in c_2) + # Check that the mutable dict 'd' has not been shallow copied + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + @raises(KeyError) + def test_raise_keyerror(self): + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + a = c['gfgd'] + + def test_revertitem(self): + c = COWDictBase() + c['xyz'] = 1 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + c_2 = c.copy() + c_2['xyz'] = 2 + self.assertTrue(c_2['xyz'], 2) + c_2.__revertitem__('xyz') + self.assertTrue(c_2['xyz'], 1) + + c_2['d']['abc'] = 20 + self.assertTrue(c_2['d']['abc'], 20) + c_2.__revertitem__('d') + self.assertTrue(c_2['d']['abc'], 10) + + def test_cowset(self): + c = COWDictBase() + c['set'] = COWSetBase() + c['set'].add("o1") + c['set'].add("o1") + self.assertTrue(len(c['set']), 1) + c['set'].add("o2") + self.assertTrue(len(c['set']), 2) + c['set'].remove("o1") + self.assertTrue(len(c['set']), 1) + self.assertTrue('o2' in c['set']) + + def test_cow_copy_anything(self): + class Anything: + var = 0 + pass + a = Anything() + c = COWDictBase() + c['any'] = a + self.assertEquals(id(a), id(getattr(c, 'any__mutable__'))) + c_2 = c.copy() + self.assertEquals(id(a), id(getattr(c_2, 'any__mutable__'))) + c_2['any'].var = 1 + self.assertNotEquals(id(a), id(getattr(c_2, 'any__mutable__'))) diff --git a/lib/bb/tests/data.py b/lib/bb/tests/test_data.py similarity index 100% rename from lib/bb/tests/data.py rename to lib/bb/tests/test_data.py diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/test_fetch.py similarity index 100% rename from lib/bb/tests/fetch.py rename to lib/bb/tests/test_fetch.py diff --git a/lib/bb/tests/utils.py b/lib/bb/tests/test_utils.py similarity index 100% rename from lib/bb/tests/utils.py rename to lib/bb/tests/test_utils.py