Skip to content

Commit

Permalink
Python: extract reference tags
Browse files Browse the repository at this point in the history 
Signed-off-by: Masatake YAMATO <[email protected]>
  • Loading branch information
@masatake
masatake committed Nov 21, 2022
1 parent 6f9a232 commit 678461c
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 7 deletions.
2 changes: 2 additions & 0 deletions Tmain/list-roles.d/stdout-expected.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ Python i/module indirectlyImported on module imported in a
Python i/module namespace on namespace from where classes/variables/functions are imported
Python x/unknown imported on imported from the other module
Python x/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
Python x/unknown ref off referenced anyhow
R l/library library on library attached by library function
R l/library require on library attached by require function
R s/source source on source loaded by source fucntion
Expand Down Expand Up @@ -208,6 +209,7 @@ Python i/module indirectlyImported on module imported in a
Python i/module namespace on namespace from where classes/variables/functions are imported
Python x/unknown imported on imported from the other module
Python x/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
Python x/unknown ref off referenced anyhow
R l/library library on library attached by library function
R l/library require on library attached by require function
R s/source source on source loaded by source fucntion
Expand Down
4 changes: 4 additions & 0 deletions Units/parser-python.r/reftag.d/args.ctags
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
--sort=no
--extras=+r
--fields=+r
--roles-Python.{unknown}=+{ref}
22 changes: 22 additions & 0 deletions Units/parser-python.r/reftag.d/expected.tags
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
a.b input.py /^import a.b$/;" i roles:imported
object input.py /^class X(object):$/;" x roles:ref
X input.py /^class X(object):$/;" c roles:def
self input.py /^ def __init__(self, n):$/;" x roles:ref
n input.py /^ def __init__(self, n):$/;" x roles:ref
__init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def
self input.py /^ self.n = n$/;" x roles:ref
n input.py /^ self.n = n$/;" x roles:ref
n input.py /^ self.n = n$/;" x roles:ref
self input.py /^ def val(self):$/;" x roles:ref
val input.py /^ def val(self):$/;" m class:X roles:def
self input.py /^ return self.n$/;" x roles:ref
n input.py /^ return self.n$/;" x roles:ref
one input.py /^def one():$/;" f roles:def
X input.py /^ return X(1)$/;" x roles:ref
two input.py /^def two():$/;" f roles:def
X input.py /^ return X(2)$/;" x roles:ref
print input.py /^print (one().val() + two().val())$/;" x roles:ref
one input.py /^print (one().val() + two().val())$/;" x roles:ref
val input.py /^print (one().val() + two().val())$/;" x roles:ref
two input.py /^print (one().val() + two().val())$/;" x roles:ref
val input.py /^print (one().val() + two().val())$/;" x roles:ref
18 changes: 18 additions & 0 deletions Units/parser-python.r/reftag.d/input.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import a.b

class X(object):
def __init__(self, n):
self.n = n

def val(self):
return self.n

def one():
return X(1)

def two():
return X(2)

print (one().val() + two().val())


65 changes: 58 additions & 7 deletions parsers/python.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ typedef enum {
} pythonModuleRole;

typedef enum {
PYTHON_UNKNOWN_REFERENCED,
PYTHON_UNKNOWN_IMPORTED,
PYTHON_UNKNOWN_INDIRECTLY_IMPORTED,
} pythonUnknownRole;
Expand Down Expand Up @@ -128,6 +129,7 @@ static roleDefinition PythonModuleRoles [] = {
};

static roleDefinition PythonUnknownRoles [] = {
{ false,"ref", "referenced anyhow" },
{ true, "imported", "imported from the other module" },
{ true, "indirectlyImported",
"classes/variables/functions/modules imported in alternative name" },
Expand Down Expand Up @@ -201,6 +203,7 @@ typedef struct {
int indent;
unsigned long lineNumber;
MIOPos filePosition;
int reftag;
} tokenInfo;

struct pythonNestingLevelUserData {
Expand Down Expand Up @@ -244,13 +247,23 @@ static accessType accessFromIdentifier (const vString *const ident,
return ACCESS_PROTECTED;
}

static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token,
static void useTokenAsPartOfTag (tokenInfo *const token)
{
if (token->reftag == CORK_NIL)
return;

markCorkEntryAsPlaceholder (token->reftag, true);
token->reftag = CORK_NIL;
}

static void initPythonEntry (tagEntryInfo *const e, tokenInfo *const token,
const pythonKind kind)
{
accessType access;
int parentKind = -1;
NestingLevel *nl;

useTokenAsPartOfTag(token);
initTagEntry (e, vStringValue (token->string), kind);

e->lineNumber = token->lineNumber;
Expand Down Expand Up @@ -284,7 +297,7 @@ static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token
e->isFileScope = true;
}

static int makeClassTag (const tokenInfo *const token,
static int makeClassTag (tokenInfo *const token,
const vString *const inheritance,
const vString *const decorators)
{
Expand All @@ -307,7 +320,7 @@ static int makeClassTag (const tokenInfo *const token,
return CORK_NIL;
}

static int makeFunctionTag (const tokenInfo *const token,
static int makeFunctionTag (tokenInfo *const token,
const vString *const arglist,
const vString *const decorators)
{
Expand All @@ -331,7 +344,7 @@ static int makeFunctionTag (const tokenInfo *const token,
return CORK_NIL;
}

static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const kind)
static int makeSimplePythonTag (tokenInfo *const token, pythonKind const kind)
{
if (PythonKinds[kind].enabled)
{
Expand All @@ -344,7 +357,7 @@ static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const k
return CORK_NIL;
}

static int makeSimplePythonRefTag (const tokenInfo *const token,
static int makeSimplePythonRefTag (tokenInfo *const token,
const vString *const altName,
pythonKind const kind,
int roleIndex, xtagType xtag)
Expand All @@ -354,6 +367,7 @@ static int makeSimplePythonRefTag (const tokenInfo *const token,
{
tagEntryInfo e;

useTokenAsPartOfTag(token);
initRefTagEntry (&e, vStringValue (altName ? altName : token->string),
kind, roleIndex);

Expand Down Expand Up @@ -393,6 +407,7 @@ static void clearPoolToken (void *data)
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
vStringClear (token->string);
token->reftag = CORK_NIL;
}

static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
Expand All @@ -403,6 +418,7 @@ static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
dest->keyword = src->keyword;
dest->indent = src->indent;
vStringCopy(dest->string, src->string);
dest->reftag = src->reftag;
}

/* Skip a single or double quoted string. */
Expand Down Expand Up @@ -479,7 +495,7 @@ static void ungetToken (tokenInfo *const token)
copyToken (NextToken, token);
}

static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces)
{
int c;
int n;
Expand Down Expand Up @@ -694,6 +710,29 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
}
}

static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
{
readTokenFullNoRefTag (token, inclWhitespaces);

if (token->type == TOKEN_IDENTIFIER
/* Don't make a ref tag for a number. */
&& (vStringLength(token->string) > 0 &&
!isdigit((unsigned char)vStringChar(token->string, 0)))
&& PythonKinds[K_UNKNOWN].enabled
&& PythonUnknownRoles[PYTHON_UNKNOWN_REFERENCED].enabled)
{
const bool in_subparser = (Lang_python != getInputLanguage());
if (in_subparser)
pushLanguage(Lang_python);

token->reftag = makeSimpleRefTag (token->string,
K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED);

if (in_subparser)
popLanguage();
}
}

static void readToken (tokenInfo *const token)
{
readTokenFull (token, false);
Expand Down Expand Up @@ -788,13 +827,18 @@ static void readQualifiedName (tokenInfo *const nameToken)
vString *qualifiedName = vStringNew ();
tokenInfo *token = newToken ();

unsigned long lineNumber = nameToken->lineNumber;
MIOPos filePosition = nameToken->filePosition;
useTokenAsPartOfTag(token);

while (nameToken->type == TOKEN_IDENTIFIER ||
nameToken->type == '.')
{
vStringCat (qualifiedName, nameToken->string);
copyToken (token, nameToken);

readToken (nameToken);
useTokenAsPartOfTag (nameToken);
}
/* put the last, non-matching, token back */
ungetToken (nameToken);
Expand All @@ -805,6 +849,13 @@ static void readQualifiedName (tokenInfo *const nameToken)

deleteToken (token);
vStringDelete (qualifiedName);

tagEntryInfo e;
initRefTagEntry(&e, vStringValue (nameToken->string),
K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED);
e.lineNumber = lineNumber;
e.filePosition = filePosition;
nameToken->reftag = makeTagEntry (&e);
}
}

Expand Down Expand Up @@ -1490,7 +1541,7 @@ static bool parseVariable (tokenInfo *const token, const pythonKind kind)

do
{
const tokenInfo *const nameToken = nameTokens[i];
tokenInfo *const nameToken = nameTokens[i];
vString **type = &(nameTypes[i++]);

readToken (token);
Expand Down

0 comments on commit 678461c

Please sign in to comment.