From 7734fde7ea16b193effef39c96585d7708dc60fa Mon Sep 17 00:00:00 2001 From: Evangelos Chaniotakis Date: Mon, 25 Mar 2024 11:54:41 -0700 Subject: [PATCH] fix: [OS-285] Complete fixes for operational state (#466) * support ocd-stack's healthcheck on new centos image * further operational state fixes * fix npe * further fixes to opstatus * further FURTHER fixes to opstatus * further fix to sdp state * fix sdpId / vcId parsing * fix raw display * remove debug dumps --------- Co-authored-by: haniotak --- CHANGES.md | 3 + backend/pom.xml | 2 +- ...iveStatusOperationalStateCacheManager.java | 28 ++- .../java/net/es/oscars/task/EsdbVlanSync.java | 2 +- .../web/rest/NsoLiveStatusController.java | 211 +++++++++--------- 5 files changed, 136 insertions(+), 110 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 625f54b1..308680d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,7 @@ # OSCARS Release Notes +### 1.2.3 +- OS-285: fix operational state bugs + ### 1.2.2 > Mar 2024 - OS-227: add an ip interface frontend diff --git a/backend/pom.xml b/backend/pom.xml index 234b9da4..4d1c2988 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -14,7 +14,7 @@ net.es.oscars backend backend - 1.2.2 + 1.2.3 OSCARS backend diff --git a/backend/src/main/java/net/es/oscars/sb/nso/LiveStatusOperationalStateCacheManager.java b/backend/src/main/java/net/es/oscars/sb/nso/LiveStatusOperationalStateCacheManager.java index cad31b37..5c262dd8 100644 --- a/backend/src/main/java/net/es/oscars/sb/nso/LiveStatusOperationalStateCacheManager.java +++ b/backend/src/main/java/net/es/oscars/sb/nso/LiveStatusOperationalStateCacheManager.java @@ -95,6 +95,10 @@ public ArrayList refreshSdp(String device, int serviceId) { ArrayList sdpsList = this.getDataLines(reply); // extract SDP data - simple line format parsing + // incoming format: + // 7141:7087 Spok 134.55.200.173 Up Up 524100 524267 + // sdpId : vcId + for (int i = 0; i < sdpsList.size(); i++) { result = new LiveStatusSdpResult(); result.setDevice(device); @@ -119,14 +123,13 @@ public ArrayList refreshSdp(String device, int serviceId) { break; } - int vcId = 0; int sdpId = 0; + int vcId = 0; try { - vcId = Integer.parseInt(sdpIdAndInfo[0]); - sdpId = Integer.parseInt(sdpInfo[0]); + sdpId = Integer.parseInt(sdpIdAndInfo[0]); + vcId = Integer.parseInt(sdpInfo[0]); } catch (NumberFormatException error) { - log.error("Couldn't parse VC/SDP ID"); - error.printStackTrace(); + log.error("Couldn't parse VC/SDP ID from "+line+" : "+error.getMessage()); } result.setVcId(vcId); result.setSdpId(sdpId); @@ -396,10 +399,19 @@ public ArrayList getDataLines(String input) { if (input == null) return null; ArrayList data = new ArrayList<>(); String[] lines = input.split("\r\n"); - Pattern regex = Pattern.compile("^[0-9]{1}"); + + // starts with a number, i.e. for something like + // 1/1/c13/1:2012 7072 7001 none 7001 none Up Up + Pattern startsWithNumber = Pattern.compile("^[0-9]{1}"); + + // some SAP lines look different and we want those too + // lag-50:3603 7072 7005 none 7005 none Up Up + Pattern startsWithLag = Pattern.compile("^lag"); + for (String line : lines) { - Matcher m = regex.matcher(line); - if (m.find()) { + Matcher numberMatch = startsWithNumber.matcher(line); + Matcher lagMatch = startsWithLag.matcher(line); + if (numberMatch.find() || lagMatch.find()) { data.add(line); } } diff --git a/backend/src/main/java/net/es/oscars/task/EsdbVlanSync.java b/backend/src/main/java/net/es/oscars/task/EsdbVlanSync.java index 49b57e12..0a9730ac 100644 --- a/backend/src/main/java/net/es/oscars/task/EsdbVlanSync.java +++ b/backend/src/main/java/net/es/oscars/task/EsdbVlanSync.java @@ -39,7 +39,7 @@ public EsdbVlanSync(Startup startup, ConnectionRepository cr, ESDBProxy esdbProx this.topologyStore = topologyStore; } - @Scheduled(initialDelay = 10000,fixedDelayString ="${esdb.vlan-sync-period}" ) + @Scheduled(initialDelay = 120000, fixedDelayString ="${esdb.vlan-sync-period}" ) public void processingLoop() { if (!esdbProperties.isEnabled()) { return; diff --git a/backend/src/main/java/net/es/oscars/web/rest/NsoLiveStatusController.java b/backend/src/main/java/net/es/oscars/web/rest/NsoLiveStatusController.java index 06d961ab..89ed008a 100644 --- a/backend/src/main/java/net/es/oscars/web/rest/NsoLiveStatusController.java +++ b/backend/src/main/java/net/es/oscars/web/rest/NsoLiveStatusController.java @@ -1,5 +1,8 @@ package net.es.oscars.web.rest; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.Builder; @@ -10,14 +13,11 @@ import net.es.oscars.resv.enums.DeploymentState; import net.es.oscars.resv.enums.State; import net.es.oscars.sb.nso.db.NsoSdpIdDAO; -import net.es.oscars.sb.nso.db.NsoSdpVcIdDAO; import net.es.oscars.sb.nso.ent.NsoSdpId; -import net.es.oscars.sb.nso.ent.NsoSdpVcId; import net.es.oscars.web.beans.NsoLiveStatusRequest; import net.es.oscars.web.beans.OperationalState; import net.es.oscars.web.beans.OperationalStateInfoResponse; import net.es.oscars.web.beans.MacInfoResponse; -import net.es.oscars.sb.nso.rest.LiveStatusLspResult; import net.es.oscars.sb.nso.rest.LiveStatusSapResult; import net.es.oscars.sb.nso.rest.LiveStatusSdpResult; import net.es.oscars.sb.nso.rest.MacInfoResult; @@ -31,13 +31,13 @@ import net.es.topo.common.devel.DevelUtils; import net.es.topo.common.dto.nso.enums.NsoVplsSdpPrecedence; +import org.checkerframework.checker.units.qual.A; import org.springframework.web.bind.annotation.*; import java.time.Instant; import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; +import static net.es.oscars.web.beans.OperationalStateInfoResponse.UpDown.UP; import static net.es.topo.common.dto.nso.enums.NsoVplsSdpPrecedence.PRIMARY; import static net.es.topo.common.dto.nso.enums.NsoVplsSdpPrecedence.SECONDARY; @@ -51,7 +51,6 @@ public class NsoLiveStatusController { private final NsoSdpIdDAO nsoSdpIdDAO; - public NsoLiveStatusController( LiveStatusOperationalStateCacheManager operationalStateCacheManager, NsoVcIdDAO nsoVcIdDAO, @@ -70,7 +69,7 @@ public MacInfoResponse getMacInfo(@RequestBody NsoLiveStatusRequest request) { // filter and extract request data RequestData requestData = getRequestData(request); - if(requestData == null) { + if (requestData == null) { log.info("Couldn't extract REST request data"); throw new NoSuchElementException(); } @@ -95,11 +94,11 @@ public MacInfoResponse getMacInfo(@RequestBody NsoLiveStatusRequest request) { .getMacs(device, serviceId, request.getRefreshIfOlderThan()).getMacInfoResult()); } else { results.add(MacInfoResult.builder() - .device(device) - .errorMessage("Not deployed") - .status(false) - .timestamp(Instant.now()) - .fdbQueryResult("Not deployed") + .device(device) + .errorMessage("Not deployed") + .status(false) + .timestamp(Instant.now()) + .fdbQueryResult("Not deployed") .build()); } } @@ -115,7 +114,7 @@ public OperationalStateInfoResponse getOperationalStateInfo(@RequestBody NsoLive // filter and extract request data RequestData requestData = getRequestData(request); - if(requestData == null) { + if (requestData == null) { log.info("Couldn't extract REST request data"); throw new NoSuchElementException(); } @@ -123,17 +122,8 @@ public OperationalStateInfoResponse getOperationalStateInfo(@RequestBody NsoLive int serviceId = requestData.getServiceId(); Connection conn = requestData.getConn(); - // this collects nsoSdpIds keyed off the device - Map> byDevice = nsoSdpIdDAO.findNsoSdpIdByConnectionId(conn.getConnectionId()) - .stream().collect( - Collectors.groupingBy( - NsoSdpId::getDevice, - Collectors.mapping( - Function.identity(), - Collectors.toSet() - ) - ) - ); + // this collects nsoSdpIds that oscars would set up, keyed off the device + List nsoSdpIds = nsoSdpIdDAO.findNsoSdpIdByConnectionId(conn.getConnectionId()); // start building REST return @@ -143,54 +133,63 @@ public OperationalStateInfoResponse getOperationalStateInfo(@RequestBody NsoLive Instant timestamp = request.getRefreshIfOlderThan(); List results = new ArrayList<>(); - Map> allSdpsForDevice = new HashMap<>(); + ArrayList allSdpsForAllDevices = new ArrayList<>(); Map> allSapsForDevice = new HashMap<>(); // Map> allLspsForDevice = new HashMap<>(); - log.debug("Run live-status request on devices for operational states"); + log.debug("Run live-status request on devices and collect operational states"); for (String device : devices) { if (conn.getState().equals(State.ACTIVE) && conn.getDeploymentState().equals(DeploymentState.DEPLOYED)) { log.info("Fetch SDPs, SAPs, and LSPs from LiveStatusCacheManager for " + device + " service id " + serviceId); + List sdpsOnDevice = operationalStateCacheManager.getSdp(device, serviceId, timestamp); + List sapsOnDevice = operationalStateCacheManager.getSap(device, serviceId, timestamp); // get SDPs, SAPs, and LSPs from cache manager - allSdpsForDevice.put(device, operationalStateCacheManager.getSdp(device, serviceId, timestamp)); - allSapsForDevice.put(device, operationalStateCacheManager.getSap(device, serviceId, timestamp)); + allSdpsForAllDevices.addAll(sdpsOnDevice); + allSapsForDevice.put(device, new ArrayList<>(sapsOnDevice)); // allLspsForDevice.put(device, operationalStateCacheManager.getLsp(device, timestamp)); - + // this raw output is the same for all SDPs on the device String sdpRaw = ""; - for (LiveStatusSdpResult sdpResult : allSdpsForDevice.get(device)) { + for (LiveStatusSdpResult sdpResult : sdpsOnDevice) { sdpRaw = sdpResult.getRaw(); } + String sapRaw = ""; for (LiveStatusSapResult sapResult : allSapsForDevice.get(device)) { - sdpRaw = sapResult.getRaw(); + sapRaw = sapResult.getRaw(); } OperationalStateInfoResult resultElement = OperationalStateInfoResult.builder() .device(device) .timestamp(timestamp) .status(true) - .raw(sdpRaw+"\n"+sapRaw) + .sdps(sdpsOnDevice) + .saps(sapsOnDevice) + .raw(sdpRaw + "\n" + sapRaw) .build(); - new OperationalStateInfoResult(); + new OperationalStateInfoResult(); results.add(resultElement); } else { results.add(OperationalStateInfoResult.builder() - .device(device) - .errorMessage("Not deployed") - .status(false) - .raw("") - .timestamp(timestamp) - .build()); + .device(device) + .errorMessage("Not deployed") + .status(false) + .sdps(new ArrayList<>()) + .saps(new ArrayList<>()) + .raw("") + .timestamp(timestamp) + .build()); } } response.setResults(results); + // dumpDebug("allsdps", allSdpsForAllDevices); + // dumpDebug("nsoSdpIds", nsoSdpIds); for (OperationalStateInfoResult result : response.getResults()) { String device = result.getDevice(); @@ -202,77 +201,73 @@ public OperationalStateInfoResponse getOperationalStateInfo(@RequestBody NsoLive endpointState = OperationalState.UP; } OperationalStateInfoResponse.UpDown operState = sapResult.getOperationalState() ? - OperationalStateInfoResponse.UpDown.UP : OperationalStateInfoResponse.UpDown.DOWN; + UP : OperationalStateInfoResponse.UpDown.DOWN; OperationalStateInfoResponse.UpDown adminState = sapResult.getAdminState() ? - OperationalStateInfoResponse.UpDown.UP : OperationalStateInfoResponse.UpDown.DOWN; + UP : OperationalStateInfoResponse.UpDown.DOWN; response.getEndpoints().add(OperationalStateInfoResponse.EndpointOpInfo.builder() - .device(device) - .vlanId(sapResult.getVlan()) - .port(sapResult.getPort()) - .operState(operState) - .adminState(adminState) - .state(endpointState) - .build()); + .device(device) + .vlanId(sapResult.getVlan()) + .port(sapResult.getPort()) + .operState(operState) + .adminState(adminState) + .state(endpointState) + .build()); } - // mapping SDPs is slightly more complicated though - Map> byFarEnd = new HashMap<>(); - - // split these by far-end - for (LiveStatusSdpResult sdpResult : allSdpsForDevice.get(device)) { - if (!byFarEnd.containsKey(sdpResult.getFarEndAddress())) { - byFarEnd.put(sdpResult.getFarEndAddress(), new HashSet<>()); + List deviceSdpIds = nsoSdpIds.stream().filter(sdpId -> sdpId.getDevice().equals(device)).toList(); + // dumpDebug("deviceSdpIds", deviceSdpIds); + + // first collect our desired SDP ids and group them by remote end + Map> byTarget = new HashMap<>(); + for (NsoSdpId nsoSdpId : deviceSdpIds) { + if (!byTarget.containsKey(nsoSdpId.getTarget())) { + byTarget.put(nsoSdpId.getTarget(), new HashSet<>()); } - byFarEnd.get(sdpResult.getFarEndAddress()).add(sdpResult); + byTarget.get(nsoSdpId.getTarget()).add(nsoSdpId); } - + // dumpDebug("byTarget on "+device, byTarget); // for each far end, make a tunnel, then we have to figure out the health of the tunnel // each tunnel is composed of a primary SDP and maybe a secondary one as well. - for (String farEnd : byFarEnd.keySet()) { - String targetDevice = farEnd; - - Set sdpOpInfos = new HashSet<>(); + for (String target : byTarget.keySet()) { Map okByPrecedence = new HashMap<>(); - - for (LiveStatusSdpResult sdpResult : byFarEnd.get(farEnd)) { - - OperationalStateInfoResponse.UpDown operState = sdpResult.getOperationalState() ? - OperationalStateInfoResponse.UpDown.UP : OperationalStateInfoResponse.UpDown.DOWN; - OperationalStateInfoResponse.UpDown adminState = sdpResult.getAdminState() ? - OperationalStateInfoResponse.UpDown.UP : OperationalStateInfoResponse.UpDown.DOWN; - - NsoVplsSdpPrecedence precedence = PRIMARY; - boolean foundIt = false; - - for (NsoSdpId nsoSdpId : byDevice.get(device)) { - if (nsoSdpId.getSdpId().equals(sdpResult.getSdpId())) { - if (nsoSdpId.getPrecedence().equals(PRIMARY.toString())) { - foundIt = true; - targetDevice = nsoSdpId.getTarget(); - break; - } else if (nsoSdpId.getPrecedence().equals(SECONDARY.toString())) { + Set sdpOpInfos = new HashSet<>(); + for (NsoSdpId nsoSdpId : byTarget.get(target)) { + log.info("checking status for nsoSdpId "+nsoSdpId.getDevice()+" "+nsoSdpId.getSdpId()); + for (LiveStatusSdpResult sdpResult : allSdpsForAllDevices) { + log.info("examining sdpResult for "+sdpResult.getDevice()+" "+sdpResult.getSdpId()); + if (sdpResult.getDevice().equals(nsoSdpId.getDevice()) && sdpResult.getSdpId().equals(nsoSdpId.getSdpId())) { + log.info("matched! "); + OperationalStateInfoResponse.UpDown operState = sdpResult.getOperationalState() ? + UP : OperationalStateInfoResponse.UpDown.DOWN; + OperationalStateInfoResponse.UpDown adminState = sdpResult.getAdminState() ? + UP : OperationalStateInfoResponse.UpDown.DOWN; + String precedenceStr = nsoSdpId.getPrecedence(); + NsoVplsSdpPrecedence precedence = PRIMARY; + if (precedenceStr.equals(SECONDARY.toString())) { precedence = SECONDARY; - foundIt = true; - targetDevice = nsoSdpId.getTarget(); - break; } + if (operState.equals(UP) && adminState.equals(UP)) { + okByPrecedence.put(precedence, true); + } else { + okByPrecedence.put(precedence, false); + } + + sdpOpInfos.add(OperationalStateInfoResponse.SdpOpInfo.builder() + .sdpId(sdpResult.getSdpId()) + .vcId(sdpResult.getVcId()) + .operState(operState) + .adminState(adminState) + .precedence(precedence) + .build()); + break; } } - if (foundIt) { - boolean sdpOk = sdpResult.getAdminState() && sdpResult.getOperationalState(); - okByPrecedence.put(precedence, sdpOk); - - sdpOpInfos.add(OperationalStateInfoResponse.SdpOpInfo.builder() - .sdpId(sdpResult.getSdpId()) - .vcId(sdpResult.getVcId()) - .operState(operState) - .adminState(adminState) - .precedence(precedence) - .build()); - } + } + // dumpDebug("sdpOpInfos", sdpOpInfos); + // dumpDebug("okByPrecedence", okByPrecedence); // the rule is... // - if the primary SDP exists and is UP the tunnel is UP @@ -291,10 +286,10 @@ public OperationalStateInfoResponse getOperationalStateInfo(@RequestBody NsoLive } } response.getTunnels().add(OperationalStateInfoResponse.TunnelOpInfo.builder() - .state(tunnelState) - .sdps(sdpOpInfos.stream().toList()) - .device(device) - .remote(targetDevice) + .state(tunnelState) + .sdps(sdpOpInfos.stream().toList()) + .device(device) + .remote(target) .build()); } @@ -392,8 +387,8 @@ private RequestData getRequestData(NsoLiveStatusRequest request) { } // extract subset - for(String device : devicesFromRest) { - if(devicesFromId.contains(device)) { + for (String device : devicesFromRest) { + if (devicesFromId.contains(device)) { devices.add(device); } } @@ -404,4 +399,20 @@ private RequestData getRequestData(NsoLiveStatusRequest request) { .conn(conn) .build(); } + + private void dumpDebug(String context, Object o) { + String pretty = null; + + try { + pretty = (new ObjectMapper()) + .registerModule(new JavaTimeModule()) + .writerWithDefaultPrettyPrinter() + .writeValueAsString(o); + } catch (JsonProcessingException ex) { + log.error(ex.getMessage()); + } + + log.info(context + "\n" + pretty); + } + }