"""A lexical analyzer class for simple shell-like syntaxes."""# Module and documentation by Eric S. Raymond, 21 Dec 1998# Input stacking and error message cleanup added by ESR, March 2000# push_source() and pop_source() made explicit by ESR, January 2001.# Posix compliance, split(), string arguments, and# iterator interface by Gustavo Niemeyer, April 2003.# changes to tokenize more like Posix shells by Vinay Sajip, July 2016.import osimport reimport sysfrom collections import dequefrom io import StringIO__all__ = ["shlex", "split", "quote", "join"]class shlex:"A lexical analyzer class for simple shell-like syntaxes."def __init__(self, instream=None, infile=None, posix=False,punctuation_chars=False):if isinstance(instream, str):instream = StringIO(instream)if instream is not None:self.instream = instreamself.infile = infileelse:self.instream = sys.stdinself.infile = Noneself.posix = posixif posix:self.eof = Noneelse:self.eof = ''self.commenters = '#'self.wordchars = ('abcdfeghijklmnopqrstuvwxyz''ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')if self.posix:self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ''ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')self.whitespace = ' \t\r\n'self.whitespace_split = Falseself.quotes = '\'"'self.escape = '\\'self.escapedquotes = '"'self.state = ' 'self.pushback = deque()self.lineno = 1self.debug = 0self.token = ''self.filestack = deque()self.source = Noneif not punctuation_chars:punctuation_chars = ''elif punctuation_chars is True:punctuation_chars = '();<>|&'self._punctuation_chars = punctuation_charsif punctuation_chars:# _pushback_chars is a push back queue used by lookahead logicself._pushback_chars = deque()# these chars added because allowed in file names, args, wildcardsself.wordchars += '~-./*?='#remove any punctuation chars from wordcharst = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))self.wordchars = self.wordchars.translate(t)@propertydef punctuation_chars(self):return self._punctuation_charsdef push_token(self, tok):"Push a token onto the stack popped by the get_token method"if self.debug >= 1:print("shlex: pushing token " + repr(tok))self.pushback.appendleft(tok)def push_source(self, newstream, newfile=None):"Push an input source onto the lexer's input source stack."if isinstance(newstream, str):newstream = StringIO(newstream)self.filestack.appendleft((self.infile, self.instream, self.lineno))self.infile = newfileself.instream = newstreamself.lineno = 1if self.debug:if newfile is not None:print('shlex: pushing to file %s' % (self.infile,))else:print('shlex: pushing to stream %s' % (self.instream,))def pop_source(self):"Pop the input source stack."self.instream.close()(self.infile, self.instream, self.lineno) = self.filestack.popleft()if self.debug:print('shlex: popping to %s, line %d' \% (self.instream, self.lineno))self.state = ' 'def get_token(self):"Get a token from the input stream (or from stack if it's nonempty)"if self.pushback:tok = self.pushback.popleft()if self.debug >= 1:print("shlex: popping token " + repr(tok))return tok# No pushback. Get a token.raw = self.read_token()# Handle inclusionsif self.source is not None:while raw == self.source:spec = self.sourcehook(self.read_token())if spec:(newfile, newstream) = specself.push_source(newstream, newfile)raw = self.get_token()# Maybe we got EOF instead?while raw == self.eof:if not self.filestack:return self.eofelse:self.pop_source()raw = self.get_token()# Neither inclusion nor EOFif self.debug >= 1:if raw != self.eof:print("shlex: token=" + repr(raw))else:print("shlex: token=EOF")return rawdef read_token(self):quoted = Falseescapedstate = ' 'while True:if self.punctuation_chars and self._pushback_chars:nextchar = self._pushback_chars.pop()else:nextchar = self.instream.read(1)if nextchar == '\n':self.lineno += 1if self.debug >= 3:print("shlex: in state %r I see character: %r" % (self.state,nextchar))if self.state is None:self.token = '' # past end of filebreakelif self.state == ' ':if not nextchar:self.state = None # end of filebreakelif nextchar in self.whitespace:if self.debug >= 2:print("shlex: I see whitespace in whitespace state")if self.token or (self.posix and quoted):break # emit current tokenelse:continueelif nextchar in self.commenters:self.instream.readline()self.lineno += 1elif self.posix and nextchar in self.escape:escapedstate = 'a'self.state = nextcharelif nextchar in self.wordchars:self.token = nextcharself.state = 'a'elif nextchar in self.punctuation_chars:self.token = nextcharself.state = 'c'elif nextchar in self.quotes:if not self.posix:self.token = nextcharself.state = nextcharelif self.whitespace_split:self.token = nextcharself.state = 'a'else:self.token = nextcharif self.token or (self.posix and quoted):break # emit current tokenelse:continueelif self.state in self.quotes:quoted = Trueif not nextchar: # end of fileif self.debug >= 2:print("shlex: I see EOF in quotes state")# XXX what error should be raised here?raise ValueError("No closing quotation")if nextchar == self.state:if not self.posix:self.token += nextcharself.state = ' 'breakelse:self.state = 'a'elif (self.posix and nextchar in self.escape and self.statein self.escapedquotes):escapedstate = self.stateself.state = nextcharelse:self.token += nextcharelif self.state in self.escape:if not nextchar: # end of fileif self.debug >= 2:print("shlex: I see EOF in escape state")# XXX what error should be raised here?raise ValueError("No escaped character")# In posix shells, only the quote itself or the escape# character may be escaped within quotes.if (escapedstate in self.quotes andnextchar != self.state and nextchar != escapedstate):self.token += self.stateself.token += nextcharself.state = escapedstateelif self.state in ('a', 'c'):if not nextchar:self.state = None # end of filebreakelif nextchar in self.whitespace:if self.debug >= 2:print("shlex: I see whitespace in word state")self.state = ' 'if self.token or (self.posix and quoted):break # emit current tokenelse:continueelif nextchar in self.commenters:self.instream.readline()self.lineno += 1if self.posix:self.state = ' 'if self.token or (self.posix and quoted):break # emit current tokenelse:continueelif self.state == 'c':if nextchar in self.punctuation_chars:self.token += nextcharelse:if nextchar not in self.whitespace:self._pushback_chars.append(nextchar)self.state = ' 'breakelif self.posix and nextchar in self.quotes:self.state = nextcharelif self.posix and nextchar in self.escape:escapedstate = 'a'self.state = nextcharelif (nextchar in self.wordchars or nextchar in self.quotesor (self.whitespace_split andnextchar not in self.punctuation_chars)):self.token += nextcharelse:if self.punctuation_chars:self._pushback_chars.append(nextchar)else:self.pushback.appendleft(nextchar)if self.debug >= 2:print("shlex: I see punctuation in word state")self.state = ' 'if self.token or (self.posix and quoted):break # emit current tokenelse:continueresult = self.tokenself.token = ''if self.posix and not quoted and result == '':result = Noneif self.debug > 1:if result:print("shlex: raw token=" + repr(result))else:print("shlex: raw token=EOF")return resultdef sourcehook(self, newfile):"Hook called on a filename to be sourced."if newfile[0] == '"':newfile = newfile[1:-1]# This implements cpp-like semantics for relative-path inclusion.if isinstance(self.infile, str) and not os.path.isabs(newfile):newfile = os.path.join(os.path.dirname(self.infile), newfile)return (newfile, open(newfile, "r"))def error_leader(self, infile=None, lineno=None):"Emit a C-compiler-like, Emacs-friendly error-message leader."if infile is None:infile = self.infileif lineno is None:lineno = self.linenoreturn "\"%s\", line %d: " % (infile, lineno)def __iter__(self):return selfdef __next__(self):token = self.get_token()if token == self.eof:raise StopIterationreturn tokendef split(s, comments=False, posix=True):"""Split the string *s* using shell-like syntax."""lex = shlex(s, posix=posix)lex.whitespace_split = Trueif not comments:lex.commenters = ''return list(lex)def join(split_command):"""Return a shell-escaped string from *split_command*."""return ' '.join(quote(arg) for arg in split_command)_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).searchdef quote(s):"""Return a shell-escaped version of the string *s*."""if not s:return "''"if _find_unsafe(s) is None:return s# use single quotes, and put single quotes into double quotes# the string $'b is then quoted as '$'"'"'b'return "'" + s.replace("'", "'\"'\"'") + "'"def _print_tokens(lexer):while 1:tt = lexer.get_token()if not tt:breakprint("Token: " + repr(tt))if __name__ == '__main__':if len(sys.argv) == 1:_print_tokens(shlex())else:fn = sys.argv[1]with open(fn) as f:_print_tokens(shlex(f, fn))
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。