/**
* Copyright (c) 2013 Puppet Labs, Inc. and other contributors, as listed below.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* itemis AG (http://www.itemis.eu) - initial API and implementation
* Puppet Labs - adaption to DomModel and Contextual formatter
*
*/
package com.puppetlabs.xtext.serializer.acceptor;
import java.util.Collections;
import java.util.List;
import java.util.ListIterator;
import java.util.Set;
import com.puppetlabs.xtext.dommodel.IDomNode;
import com.puppetlabs.xtext.serializer.ICommentReconcilement;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.xtext.AbstractRule;
import org.eclipse.xtext.Action;
import org.eclipse.xtext.CrossReference;
import org.eclipse.xtext.Grammar;
import org.eclipse.xtext.GrammarUtil;
import org.eclipse.xtext.Keyword;
import org.eclipse.xtext.ParserRule;
import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.nodemodel.BidiTreeIterator;
import org.eclipse.xtext.nodemodel.ICompositeNode;
import org.eclipse.xtext.nodemodel.ILeafNode;
import org.eclipse.xtext.nodemodel.INode;
import org.eclipse.xtext.nodemodel.util.NodeModelUtils;
import org.eclipse.xtext.parsetree.reconstr.IHiddenTokenHelper;
import org.eclipse.xtext.parsetree.reconstr.impl.NodeIterator;
import org.eclipse.xtext.parsetree.reconstr.impl.TokenUtil;
import org.eclipse.xtext.serializer.acceptor.ISequenceAcceptor;
import org.eclipse.xtext.serializer.acceptor.ISyntacticSequenceAcceptor;
import org.eclipse.xtext.serializer.diagnostic.ISerializationDiagnostic.Acceptor;
import org.eclipse.xtext.serializer.sequencer.ISyntacticSequencer;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
/**
* This is an adapted version of HiddenTokenSequencer that emits implicit white space where it is allowed.
* Implicit WS is emitted also when an INode model is not present.
*
*/
public class HiddenTokenSequencer implements IHiddenTokenSequencer2, ISyntacticSequenceAcceptor {
@Inject
protected IHiddenTokenHelper hiddenTokenHelper;
@Inject
protected TokenUtil tokenUtil;
protected ISequenceAcceptor delegate;
protected INode lastNode;
protected INode lastNodes[] = new INode[2];
protected INode rootNode;
protected ISyntacticSequencer sequencer;
/**
* List of 'tokens' that are currently hidden.
*/
protected List<AbstractRule> currentHidden;
/**
* Stack tracking the state of 'hidden tokens'
*/
protected List<List<AbstractRule>> hiddenStack = Lists.newArrayList();
protected List<RuleCall> stack = Lists.newArrayList();
/**
* When searching for hidden nodes between INodes 'from' and 'to', the {@link #hiddenInLastNode} describes
* the state of 'hidden' when 'from' was sequenced, and {@link #currentHidden} when 'to' is sequenced.
*/
protected List<AbstractRule> hiddenInLastNode;
private ICommentReconcilement commentReconcilement;
@Override
public void acceptAssignedCrossRefDatatype(RuleCall rc, String tkn, EObject val, int index, ICompositeNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberNode(node);
delegate.acceptAssignedCrossRefDatatype(rc, tkn, val, index, node);
}
@Override
public void acceptAssignedCrossRefEnum(RuleCall rc, String token, EObject value, int index, ICompositeNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberLastLeaf(node);
delegate.acceptAssignedCrossRefEnum(rc, token, value, index, node);
}
@Override
public void acceptAssignedCrossRefKeyword(Keyword kw, String token, EObject value, int index, ILeafNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberLastLeaf(node);
delegate.acceptAssignedCrossRefKeyword(kw, token, value, index, node);
}
@Override
public void acceptAssignedCrossRefTerminal(RuleCall rc, String token, EObject value, int index, ILeafNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberNode(node);
delegate.acceptAssignedCrossRefTerminal(rc, token, value, index, node);
}
@Override
public void acceptAssignedDatatype(RuleCall rc, String token, Object value, int index, ICompositeNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberLastLeaf(node);
delegate.acceptAssignedDatatype(rc, token, value, index, node);
}
@Override
public void acceptAssignedEnum(RuleCall enumRC, String token, Object value, int index, ICompositeNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberLastLeaf(node);
delegate.acceptAssignedEnum(enumRC, token, value, index, node);
}
@Override
public void acceptAssignedKeyword(Keyword keyword, String token, Object value, int index, ILeafNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberNode(node);
delegate.acceptAssignedKeyword(keyword, token, value, index, node);
}
@Override
public void acceptAssignedTerminal(RuleCall terminalRC, String token, Object value, int index, ILeafNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberNode(node);
delegate.acceptAssignedTerminal(terminalRC, token, value, index, node);
}
@Override
public void acceptUnassignedAction(Action action) {
delegate.acceptUnassignedAction(action);
}
@Override
public void acceptUnassignedDatatype(RuleCall datatypeRC, String token, ICompositeNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberLastLeaf(node);
delegate.acceptUnassignedDatatype(datatypeRC, token, node);
}
@Override
public void acceptUnassignedEnum(RuleCall enumRC, String token, ICompositeNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberLastLeaf(node);
delegate.acceptUnassignedEnum(enumRC, token, node);
}
@Override
public void acceptUnassignedKeyword(Keyword keyword, String token, ILeafNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberNode(node);
delegate.acceptUnassignedKeyword(keyword, token, node);
}
@Override
public void acceptUnassignedTerminal(RuleCall terminalRC, String token, ILeafNode node) {
emitHiddenTokens(getHiddenNodesBetween(lastNode, node));
rememberNode(node);
delegate.acceptUnassignedTerminal(terminalRC, token, node);
}
protected void emitHiddenTokens(List<INode> hiddens /* Set<INode> comments, */) {
if(hiddens == null)
return;
boolean lastNonWhitespace = true;
AbstractRule ws = hiddenTokenHelper.getWhitespaceRuleFor(null, "");
INode prevCommentNode = null;
for(INode node : hiddens)
if(tokenUtil.isCommentNode(node)) {
if(lastNonWhitespace) {
String wsString = commentReconcilement != null
? commentReconcilement.getWhitespaceBetween(prevCommentNode, node)
: "";
delegate.acceptWhitespace(hiddenTokenHelper.getWhitespaceRuleFor(null, wsString), wsString, null);
}
delegate.acceptComment((AbstractRule) node.getGrammarElement(), node.getText(), (ILeafNode) node);
lastNonWhitespace = true;
prevCommentNode = node;
}
else {
delegate.acceptWhitespace((AbstractRule) node.getGrammarElement(), node.getText(), (ILeafNode) node);
lastNonWhitespace = false;
}
// NOTE: The original implementation has a FIX-ME note here that whitespace should be determined
// correctly. (Well, it did not work until a check was added if the ws was hidden or not).
// Longer explanation:
// When there is no WS between two elements and no node model the contextual serializer/formatter
// performs serialization by inserting an IMPLICIT WS.
// When the node model is created, empty ws nodes are skipped, and thus have to be created (this happens
// here). The created whitespace node should *NOT* be marked as implicit, since it by virtue of having been
// parsed is now the source text and should not be subject to formatting (like the IMPLICIT WS always is)
// when in "preserve whitespace" mode.
// Finally, what the fix below does is to also check if a missing WS should be emitted based on
// if whitespace is visible or not.
// THIS IS PROBABLY STILL NOT ENOUGH, as it may overrule the attempt to treat visible WS as eligible for formatting
// see isImpliedWhitespace and where it is called.
// if(lastNonWhitespace && currentHidden.contains(ws)) {
if(lastNonWhitespace && (hiddenInLastNode.contains(ws) || currentHidden.contains(ws))) {
// hiddenInLastNode.contains(ws)) {
delegate.acceptWhitespace(ws, "", null);
}
}
public boolean enterAssignedAction(Action action, EObject semanticChild, ICompositeNode node) {
return delegate.enterAssignedAction(action, semanticChild, node);
}
public boolean enterAssignedParserRuleCall(RuleCall rc, EObject semanticChild, ICompositeNode node) {
push(rc);
return delegate.enterAssignedParserRuleCall(rc, semanticChild, node);
}
public void enterUnassignedParserRuleCall(RuleCall rc) {
push(rc);
delegate.enterUnassignedParserRuleCall(rc);
}
public void finish() {
if(rootNode != null && rootNode == rootNode.getRootNode()) {
List<INode> hidden = getRemainingHiddenNodesInContainer(lastNode, rootNode);
if(!hidden.isEmpty()) {
emitHiddenTokens(hidden);
lastNodes[1] = lastNodes[0];
lastNode = lastNodes[0] = rootNode;
}
}
delegate.finish();
}
protected Set<INode> getCommentsForEObject(EObject semanticObject, INode node) {
if(node == null)
return Collections.emptySet();
Set<INode> result = Sets.newHashSet();
BidiTreeIterator<INode> ti = node.getAsTreeIterable().iterator();
while(ti.hasNext()) {
INode next = ti.next();
if(next.getSemanticElement() != null && next.getSemanticElement() != semanticObject) {
ti.prune();
continue;
}
if(tokenUtil.isCommentNode(next))
result.add(next);
}
return result;
}
protected List<INode> getHiddenNodesBetween(INode from, INode to) {
List<INode> result = getHiddenNodesBetween2(from, to);
result = reconcileComments(result, lastNodes[1], from, to);
if(result == null) {
AbstractRule ws = hiddenTokenHelper.getWhitespaceRuleFor(null, "");
// only emit hidden whitespace, or visible whitespace where this is overridden using
// isImpliedWhitespace
// boolean implied = currentHidden != null && currentHidden.contains(ws);
boolean impliedFrom = hiddenInLastNode != null && hiddenInLastNode.contains(ws);
boolean impliedTo = currentHidden != null && currentHidden.contains(ws);
boolean implied = impliedFrom || impliedTo;
int sz = stack.size();
implied = isImpliedWhitespace(implied, sz == 0
? null
: stack.get(sz - 1), from, to);
if(implied) {
delegate.acceptWhitespace(ws, IDomNode.IMPLIED_EMPTY_WHITESPACE, null);
}
}
return result;
}
protected List<INode> getHiddenNodesBetween2(INode from, INode to) {
if(from == null || to == null)
return null;
List<INode> out = Lists.newArrayList();
NodeIterator ni = new NodeIterator(from);
while(ni.hasNext()) {
INode next = ni.next();
if(tokenUtil.isWhitespaceOrCommentNode(next)) {
out.add(next);
}
else if(next.equals(to)) {
if(next instanceof ICompositeNode &&
(GrammarUtil.isDatatypeRuleCall(next.getGrammarElement()) ||
GrammarUtil.isEnumRuleCall(next.getGrammarElement()) || next.getGrammarElement() instanceof CrossReference))
while(ni.hasNext()) {
INode next2 = ni.next();
if(tokenUtil.isWhitespaceOrCommentNode(next2)) {
out.add(next2);
}
else if(next2 instanceof ILeafNode)
return out;
}
else
return out;
}
else if(tokenUtil.isToken(next))
return out; // null;
}
return out;
}
private INode getLastLeaf(INode node) {
while(node instanceof ICompositeNode)
node = ((ICompositeNode) node).getLastChild();
return node;
}
protected List<INode> getRemainingHiddenNodesInContainer(INode from, INode root) {
if(from == null || root == null)
return Collections.emptyList();
List<INode> out = Lists.newArrayList();
NodeIterator ni = new NodeIterator(from);
while(ni.hasNext()) {
INode next = ni.next();
if(next.getTotalOffset() > root.getTotalEndOffset())
return out;
else if(tokenUtil.isWhitespaceOrCommentNode(next)) {
out.add(next);
}
else if(tokenUtil.isToken(next))
return Collections.emptyList();
}
return out;
}
public void init(EObject context, EObject semanticObject, ISequenceAcceptor sequenceAcceptor, Acceptor errorAcceptor) {
init(context, semanticObject, sequenceAcceptor, errorAcceptor, null);
}
@Override
public void init(EObject context, EObject semanticObject, ISequenceAcceptor sequenceAcceptor,
Acceptor errorAcceptor, ICommentReconcilement commentReconciliator) {
this.delegate = sequenceAcceptor;
this.lastNode = NodeModelUtils.findActualNodeFor(semanticObject);
this.rootNode = lastNode;
lastNodes[0] = lastNodes[1] = this.lastNode;
initCurrentHidden(context);
this.commentReconcilement = commentReconciliator;
}
protected void initCurrentHidden(EObject context) {
// when called for a specific parser rule, its hidden() spec (if any) is made current
// otherwise the hidden() spec of the grammar is made current.
// (There is no real way to calculate the calling chain to a particular starting parser rule)
//
if(context instanceof ParserRule) {
ParserRule pr = (ParserRule) context;
if(pr.isDefinesHiddenTokens())
currentHidden = pr.getHiddenTokens();
else {
Grammar grammar = GrammarUtil.getGrammar(context);
currentHidden = grammar.getHiddenTokens();
}
// TODO: Verify this is correct
hiddenInLastNode = currentHidden;
}
}
/**
* This method should be overridden in an implementation where certain visible whitespace
* rules should be subject to formatting.
*
* @param defaultResult
* - the result to return if the already made decision is ok
* @param rc
* - the {@link RuleCall} (or {@link Grammar}) in the call chain that determined what is hidden
* @param from
* - the node left of where the ws appears, or null if there is no node model
* @param to
* - the node right if where the ws appears, or null if there is no node model
* @return true if this WS should be eligible for formatting
*/
protected boolean isImpliedWhitespace(boolean defaultResult, EObject rc, INode from, INode to) {
return defaultResult;
}
public void leaveAssignedAction(Action action, EObject semanticChild) {
delegate.leaveAssignedAction(action, semanticChild);
}
public void leaveAssignedParserRuleCall(RuleCall rc, EObject semanticChild) {
delegate.leaveAssignedParserRuleCall(rc, semanticChild);
pop();
}
public void leaveUnssignedParserRuleCall(RuleCall rc) {
delegate.leaveUnssignedParserRuleCall(rc);
pop();
}
protected void pop() {
RuleCall top = stack.remove(stack.size() - 1);
// if the rule call on top defines hidden, it pushed on the hidden stack, and state needs to
// be restored
final AbstractRule r = top.getRule();
if(r instanceof ParserRule && ((ParserRule) r).isDefinesHiddenTokens()) {
currentHidden = hiddenStack.remove(hiddenStack.size() - 1);
}
}
protected void push(RuleCall rc) {
stack.add(rc);
// if rule defines hidden, remember previous hidden, and set the new as current
final AbstractRule r = rc.getRule();
if(r instanceof ParserRule && ((ParserRule) r).isDefinesHiddenTokens()) {
hiddenStack.add(currentHidden);
currentHidden = ((ParserRule) r).getHiddenTokens();
}
}
protected List<INode> reconcileComments(List<INode> currentResult, INode preceding, INode from, INode to) {
if(preceding == null || from == null || to == null || commentReconcilement == null)
return currentResult;
List<INode> result = commentReconcilement.commentNodesFor(preceding, from, to);
// If the normal comment hunt includes a comment - remove it if it is reconciled.
// It should perhaps not appear in that position
//
if(currentResult != null) {
{
ListIterator<INode> litor = currentResult.listIterator();
while(litor.hasNext()) {
INode n = litor.next();
if(tokenUtil.isCommentNode(n) && commentReconcilement.isReconciledCommentNode(n))
litor.remove();
}
}
// Add the comment nodes the reconciler has for this position
// (possibly something just removed - that is expected)
// Also, remove whitespace from current result, if there is any reconciled comments
if(result.size() > 0) {
ListIterator<INode> litor = currentResult.listIterator();
while(litor.hasNext()) {
INode n = litor.next();
if(tokenUtil.isWhitespaceNode(n))
litor.remove();
}
}
for(INode n : result)
currentResult.add(n);
result = currentResult;
}
return result;
}
/**
* Remembers the last leaf of given node unless it is null.
*
* @param nodeToRemember
*/
private void rememberLastLeaf(INode node) {
if(node == null)
return;
lastNodes[1] = lastNodes[0];
lastNode = lastNodes[0] = getLastLeaf(node);
hiddenInLastNode = currentHidden;
}
/**
* Remembers the given node unless it is null.
*
* @param nodeToRemember
*/
private void rememberNode(INode nodeToRemember) {
if(nodeToRemember == null)
return;
lastNodes[1] = lastNodes[0];
lastNode = lastNodes[0] = nodeToRemember;
hiddenInLastNode = currentHidden;
}
}